From 8e379a8d7a45467b4b4d4034a8a336fc55b31e0a Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 22 Sep 2025 21:03:09 +0300 Subject: [PATCH 1/4] Central driver POC https://github.com/kubeflow/pipelines/pull/12023 - Modify Argo compiler: generate a plugin template instead of a container - driver as a http server Signed-off-by: arpechenin --- backend/Dockerfile.driver | 2 +- backend/src/driver/api/request.go | 44 + backend/src/driver/api/response.go | 20 + .../{v2/cmd => }/driver/execution_paths.go | 0 backend/src/driver/main.go | 178 ++++ backend/src/{v2/cmd => }/driver/main_test.go | 0 backend/src/driver/rpc_handler.go | 285 +++++ backend/src/v2/cmd/driver/main.go | 341 ------ backend/src/v2/compiler/argocompiler/argo.go | 2 +- .../src/v2/compiler/argocompiler/container.go | 91 +- backend/src/v2/compiler/argocompiler/dag.go | 90 +- .../src/v2/compiler/argocompiler/plugin.go | 22 + .../base/pipeline/kustomization.yaml | 1 + .../ml-pipeline-apiserver-deployment.yaml | 2 +- .../ml-pipeline-driver-plugin-cm.yaml | 23 + .../base/pipeline/pipeline-runner-role.yaml | 51 +- .../base/pipeline/pipeline-runner-sa.yaml | 9 + .../kustomize/env/dev/kustomization.yaml | 4 +- .../application-controller-deployment.yaml | 4 +- .../workflow-controller-deployment-patch.yaml | 3 + .../arguments-parameters.yaml | 533 +++++----- .../arguments.pipeline.yaml | 548 +++++----- .../compiled-workflows/artifact_cache.yaml | 701 ++++++------- .../container_component_with_no_inputs.yaml | 534 +++++----- .../compiled-workflows/container_io.yaml | 535 +++++----- test_data/compiled-workflows/dict_input.yaml | 545 +++++----- .../importer_pipeline_compiled.yaml | 751 ++++++------- ...sted_pipeline_opt_inputs_nil_compiled.yaml | 787 +++++++------- .../compiled-workflows/parameter_cache.yaml | 701 ++++++------- .../pipeline_in_pipeline_complex.yaml | 989 +++++++++--------- .../pipeline_with_exit_handler.yaml | 707 ++++++------- .../pipeline_with_input_status_state.yaml | 671 ++++++------ .../pipeline_with_secret_as_env.yaml | 613 ++++++----- .../pythonic_artifact_with_single_return.yaml | 236 +++-- 34 files changed, 5016 insertions(+), 5007 deletions(-) create mode 100644 backend/src/driver/api/request.go create mode 100644 backend/src/driver/api/response.go rename backend/src/{v2/cmd => }/driver/execution_paths.go (100%) create mode 100644 backend/src/driver/main.go rename backend/src/{v2/cmd => }/driver/main_test.go (100%) create mode 100644 backend/src/driver/rpc_handler.go delete mode 100644 backend/src/v2/cmd/driver/main.go create mode 100644 backend/src/v2/compiler/argocompiler/plugin.go create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver index e51c6d6c63d..4912857f471 100644 --- a/backend/Dockerfile.driver +++ b/backend/Dockerfile.driver @@ -27,7 +27,7 @@ RUN GO111MODULE=on go mod download COPY . . -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/driver/*.go FROM alpine:3.19 diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go new file mode 100644 index 00000000000..08c8360b381 --- /dev/null +++ b/backend/src/driver/api/request.go @@ -0,0 +1,44 @@ +package api + +type DriverPluginArgs struct { + CachedDecisionPath string `json:"cached_decision_path"` + Component string `json:"component,omitempty"` + Container string `json:"container,omitempty"` + RunMetadata string `json:"run_metadata,omitempty"` + DagExecutionID string `json:"dag_execution_id"` + IterationIndex string `json:"iteration_index"` + HttpProxy string `json:"http_proxy"` + HttpsProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` + KubernetesConfig string `json:"kubernetes_config,omitempty"` + RuntimeConfig string `json:"runtime_config,omitempty"` + PipelineName string `json:"pipeline_name"` + RunID string `json:"run_id"` + RunName string `json:"run_name"` + RunDisplayName string `json:"run_display_name"` + TaskName string `json:"task_name"` + Task string `json:"task"` + Type string `json:"type"` + CacheDisabledFlag bool `json:"cache_disabled"` + PublishLogs string `json:"publish_logs"` + ExecutionIdPath string `json:"execution_id_path"` + IterationCountPath string `json:"iteration_count_path"` + ConditionPath string `json:"condition_path"` + PodSpecPathPath string `json:"pod_spec_patch_path"` +} + +type DriverPlugin struct { + DriverPlugin *DriverPluginContainer `json:"driver-plugin"` +} + +type DriverPluginContainer struct { + Args *DriverPluginArgs `json:"args"` +} + +type DriverTemplate struct { + Plugin *DriverPlugin `json:"plugin"` +} + +type DriverRequest struct { + Template *DriverTemplate `json:"template"` +} diff --git a/backend/src/driver/api/response.go b/backend/src/driver/api/response.go new file mode 100644 index 00000000000..fb571609522 --- /dev/null +++ b/backend/src/driver/api/response.go @@ -0,0 +1,20 @@ +package api + +type DriverResponse struct { + Node Node `json:"node"` +} + +type Node struct { + Phase string `json:"phase"` + Outputs Outputs `json:"outputs"` + Message string `json:"message"` +} + +type Outputs struct { + Parameters []Parameter `json:"parameters"` +} + +type Parameter struct { + Name string `json:"name"` + Value string `json:"value"` +} diff --git a/backend/src/v2/cmd/driver/execution_paths.go b/backend/src/driver/execution_paths.go similarity index 100% rename from backend/src/v2/cmd/driver/execution_paths.go rename to backend/src/driver/execution_paths.go diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go new file mode 100644 index 00000000000..985fa224488 --- /dev/null +++ b/backend/src/driver/main.go @@ -0,0 +1,178 @@ +// Copyright 2021-2023 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/http" + + "google.golang.org/protobuf/encoding/protojson" + + "github.com/kubeflow/pipelines/backend/src/common/util" + + "os" + "path/filepath" + "strconv" + + "github.com/golang/glog" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" + "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" +) + +const ( + unsetProxyArgValue = "unset" + ROOT_DAG = "ROOT_DAG" + DAG = "DAG" + CONTAINER = "CONTAINER" +) + +var ( + logLevel = flag.String("log_level", "1", "The verbosity level to log.") + + // config + mlmdServerAddress = flag.String("mlmd_server_address", "metadata-grpc-service", "MLMD server address") + mlmdServerPort = flag.String("mlmd_server_port", "8080", "MLMD server port") + + serverPort = flag.String("server_port", ":8080", "Server port") +) + +func main() { + flag.Parse() + + glog.Infof("Setting log level to: '%s'", *logLevel) + err := flag.Set("v", *logLevel) + if err != nil { + glog.Warningf("Failed to set log level: %s", err.Error()) + } + + http.HandleFunc("/api/v1/template.execute", ExecutePlugin) + glog.Infof("Server started at http://localhost%v", *serverPort) + err = http.ListenAndServe(*serverPort, nil) + if err != nil { + glog.Warningf("Failed to start http server: %s", err.Error()) + } +} + +// Use WARNING default logging level to facilitate troubleshooting. +func init() { + flag.Set("logtostderr", "true") + // Change the WARNING to INFO level for debugging. + flag.Set("stderrthreshold", "WARNING") +} + +func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { + var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig + if *k8sExecConfigJson != "" { + glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) + k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} + if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) + } + } + return k8sExecCfg, nil +} + +func handleExecution(execution *driver.Execution, driverType string, executionPaths *ExecutionPaths) error { + if execution.ID != 0 { + glog.Infof("output execution.ID=%v", execution.ID) + if executionPaths.ExecutionID != "" { + if err := writeFile(executionPaths.ExecutionID, []byte(fmt.Sprint(execution.ID))); err != nil { + return fmt.Errorf("failed to write execution ID to file: %w", err) + } + } + } + if execution.IterationCount != nil { + if err := writeFile(executionPaths.IterationCount, []byte(fmt.Sprintf("%v", *execution.IterationCount))); err != nil { + return fmt.Errorf("failed to write iteration count to file: %w", err) + } + } else { + if driverType == ROOT_DAG { + if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { + return fmt.Errorf("failed to write iteration count to file: %w", err) + } + } + } + if execution.Cached != nil { + if err := writeFile(executionPaths.CachedDecision, []byte(strconv.FormatBool(*execution.Cached))); err != nil { + return fmt.Errorf("failed to write cached decision to file: %w", err) + } + } + if execution.Condition != nil { + if err := writeFile(executionPaths.Condition, []byte(strconv.FormatBool(*execution.Condition))); err != nil { + return fmt.Errorf("failed to write condition to file: %w", err) + } + } else { + // nil is a valid value for Condition + if driverType == ROOT_DAG || driverType == CONTAINER { + if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { + return fmt.Errorf("failed to write condition to file: %w", err) + } + } + } + if execution.PodSpecPatch != "" { + glog.Infof("output podSpecPatch=\n%s\n", execution.PodSpecPatch) + if executionPaths.PodSpecPatch == "" { + return fmt.Errorf("--pod_spec_patch_path is required for container executor drivers") + } + if err := writeFile(executionPaths.PodSpecPatch, []byte(execution.PodSpecPatch)); err != nil { + return fmt.Errorf("failed to write pod spec patch to file: %w", err) + } + } + if execution.ExecutorInput != nil { + executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) + if err != nil { + return fmt.Errorf("failed to marshal ExecutorInput to JSON: %w", err) + } + executorInputJSON := string(executorInputBytes) + glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) + } + return nil +} + +func prettyPrint(jsonStr string) string { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") + if err != nil { + return jsonStr + } + return prettyJSON.String() +} + +func writeFile(path string, data []byte) (err error) { + if path == "" { + return fmt.Errorf("path is not specified") + } + defer func() { + if err != nil { + err = fmt.Errorf("failed to write to %s: %w", path, err) + } + }() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func newMlmdClient() (*metadata.Client, error) { + mlmdConfig := metadata.DefaultConfig() + if *mlmdServerAddress != "" && *mlmdServerPort != "" { + mlmdConfig.Address = *mlmdServerAddress + mlmdConfig.Port = *mlmdServerPort + } + return metadata.NewClient(mlmdConfig.Address, mlmdConfig.Port) +} diff --git a/backend/src/v2/cmd/driver/main_test.go b/backend/src/driver/main_test.go similarity index 100% rename from backend/src/v2/cmd/driver/main_test.go rename to backend/src/driver/main_test.go diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go new file mode 100644 index 00000000000..d80d2180497 --- /dev/null +++ b/backend/src/driver/rpc_handler.go @@ -0,0 +1,285 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/golang/glog" + "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/kubeflow/pipelines/backend/src/driver/api" + "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" + "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "google.golang.org/protobuf/encoding/protojson" + "io" + "net/http" + "strconv" +) + +func ExecutePlugin(w http.ResponseWriter, r *http.Request) { + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + glog.Errorf("Error closing response body: %v", err) + } + }(r.Body) + + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + args, err := parseDriverRequestArgs(r) + if err != nil { + glog.Errorf("Failed to parse driver request args: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if args == nil { + glog.Errorf("Failed to parse driver request args: nil") + http.Error(w, "Driver plugin requires at least one argument", http.StatusBadRequest) + return + } + glog.Infof("driver plugin arguments: %v", args) + execution, err := drive(*args) + outputs := extractOutputParameters(execution, args.Type) + if err != nil { + glog.Errorf("unable to drive execution: %v", err) + resp := api.DriverResponse{ + Node: api.Node{ + Phase: "Failed", + Outputs: api.Outputs{ + Parameters: outputs, + }, + Message: fmt.Sprintf("unable to drive execution: %v", err), + }, + } + WriteJSONResponse(w, resp) + return + } + if execution != nil && execution.ExecutorInput != nil { + executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) + if err != nil { + WriteJSONResponse(w, api.DriverResponse{ + Node: api.Node{ + Phase: "Failed", + Outputs: api.Outputs{ + Parameters: outputs, + }, + Message: fmt.Sprintf("unable to drive execution: failed to marshal ExecutorInput to JSON: %v", err), + }, + }) + return + } + executorInputJSON := string(executorInputBytes) + glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) + } + resp := api.DriverResponse{ + Node: api.Node{ + Phase: "Succeeded", + Outputs: api.Outputs{ + Parameters: outputs, + }, + }, + } + WriteJSONResponse(w, resp) +} + +func parseDriverRequestArgs(r *http.Request) (*api.DriverPluginArgs, error) { + var body api.DriverRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("failed to parse driver request body: %v", err) + } + if body.Template == nil { + return nil, fmt.Errorf("driver request body.Template is empty") + } else if body.Template.Plugin == nil { + return nil, fmt.Errorf("driver request body.Template.Plugin is empty") + } else if body.Template.Plugin.DriverPlugin == nil { + return nil, fmt.Errorf("driver request body.Template.Plugin.DriverPlugin is empty") + } else if body.Template.Plugin.DriverPlugin.Args == nil { + return nil, fmt.Errorf("driver request body.Template.Plugin.Args is empty") + } + args := body.Template.Plugin.DriverPlugin.Args + if err := validate(*args); err != nil { + return nil, err + } + return body.Template.Plugin.DriverPlugin.Args, nil +} + +func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("KFP driver: %w", err) + } + }() + ctx := context.Background() + + proxy.InitializeConfig(args.HttpProxy, args.HttpsProxy, args.NoProxy) + + glog.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) + componentSpec := &pipelinespec.ComponentSpec{} + if err := util.UnmarshalString(args.Component, componentSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(args.Component)) + } + var taskSpec *pipelinespec.PipelineTaskSpec + if args.Task != "" { + glog.Infof("input TaskSpec:%s\n", prettyPrint(args.Task)) + taskSpec = &pipelinespec.PipelineTaskSpec{} + if err := util.UnmarshalString(args.Task, taskSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, args.Task) + } + } + + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} + if args.Container != "" { + glog.Infof("input ContainerSpec:%s\n", prettyPrint(args.Container)) + if err := util.UnmarshalString(args.Container, containerSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, args.Container) + } + } + var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig + if args.RuntimeConfig != "" { + glog.Infof("input RuntimeConfig:%s\n", prettyPrint(args.RuntimeConfig)) + runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} + if err := util.UnmarshalString(args.RuntimeConfig, runtimeConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, args.RuntimeConfig) + } + } + k8sExecCfg, err := parseExecConfigJson(&args.KubernetesConfig) + if err != nil { + return nil, err + } + namespace, err := config.InPodNamespace() + if err != nil { + return nil, err + } + client, err := newMlmdClient() + if err != nil { + return nil, err + } + cacheClient, err := cacheutils.NewClient(args.CacheDisabledFlag) + if err != nil { + return nil, err + } + + dagExecutionID, err := strconv.ParseInt(args.DagExecutionID, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse dag execution id, error: %w", err) + } + iterationIndex, err := strconv.Atoi(args.IterationIndex) + if err != nil { + return nil, fmt.Errorf("failed to parse iteration index, error: %w", err) + } + options := driver.Options{ + PipelineName: args.PipelineName, + RunID: args.RunID, + RunName: args.RunName, + RunDisplayName: args.RunDisplayName, + Namespace: namespace, + Component: componentSpec, + Task: taskSpec, + DAGExecutionID: dagExecutionID, + IterationIndex: iterationIndex, + PublishLogs: args.PublishLogs, + CacheDisabled: args.CacheDisabledFlag, + DriverType: args.Type, + TaskName: args.TaskName, + } + var driverErr error + switch args.Type { + case ROOT_DAG: + options.RuntimeConfig = runtimeConfig + execution, driverErr = driver.RootDAG(ctx, options, client) + case DAG: + execution, driverErr = driver.DAG(ctx, options, client) + case CONTAINER: + options.Container = containerSpec + options.KubernetesExecutorConfig = k8sExecCfg + execution, driverErr = driver.Container(ctx, options, client, cacheClient) + default: + err = fmt.Errorf("unknown driverType %s", args.Type) + } + if driverErr != nil { + if execution == nil { + return nil, driverErr + } + defer func() { + // Override error with driver error, because driver error is more important. + // However, we continue running, because the following code prints debug info that + // may be helpful for figuring out why this failed. + err = driverErr + }() + } + + return execution, nil +} + +func validate(args api.DriverPluginArgs) error { + if args.Type == "" { + return fmt.Errorf("argument type must be specified") + } + if args.HttpProxy == unsetProxyArgValue { + return fmt.Errorf("argument http_proxy is required but can be an empty value") + } + if args.HttpsProxy == unsetProxyArgValue { + return fmt.Errorf("argument https_proxy is required but can be an empty value") + } + if args.NoProxy == unsetProxyArgValue { + return fmt.Errorf("argument no_proxy is required but can be an empty value") + } + // validation responsibility lives in driver itself, so we do not validate all other args + return nil +} + +func extractOutputParameters(execution *driver.Execution, driverType string) []api.Parameter { + if execution == nil { + return []api.Parameter{} + } + var outputs []api.Parameter + if execution.ID != 0 { + outputs = append(outputs, api.Parameter{ + Name: "execution-id", + Value: fmt.Sprint(execution.ID), + }) + } + if execution.IterationCount != nil { + outputs = append(outputs, api.Parameter{ + Name: "iteration-count", + Value: fmt.Sprint(execution.IterationCount), + }) + } else { + if driverType == ROOT_DAG { + outputs = append(outputs, api.Parameter{ + Name: "iteration-count", + Value: fmt.Sprint(0), + }) + } + } + if execution.Cached != nil { + outputs = append(outputs, api.Parameter{ + Name: "cached-decision", + Value: strconv.FormatBool(*execution.Cached), + }) + } + if execution.Condition != nil { + outputs = append(outputs, api.Parameter{ + Name: "condition", + Value: strconv.FormatBool(*execution.Condition), + }) + } + outputs = append(outputs, api.Parameter{ + Name: "pod-spec-patch", + Value: execution.PodSpecPatch, + }) + return outputs +} + +func WriteJSONResponse(w http.ResponseWriter, payload api.DriverResponse) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } +} diff --git a/backend/src/v2/cmd/driver/main.go b/backend/src/v2/cmd/driver/main.go deleted file mode 100644 index 55626450b19..00000000000 --- a/backend/src/v2/cmd/driver/main.go +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2021-2023 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main - -import ( - "bytes" - "context" - "encoding/json" - "flag" - "fmt" - - "google.golang.org/protobuf/encoding/protojson" - - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/common/util" - - "os" - "path/filepath" - "strconv" - - "github.com/golang/glog" - "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" - "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/driver" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" -) - -const ( - driverTypeArg = "type" - httpProxyArg = "http_proxy" - httpsProxyArg = "https_proxy" - noProxyArg = "no_proxy" - unsetProxyArgValue = "unset" - ROOT_DAG = "ROOT_DAG" - DAG = "DAG" - CONTAINER = "CONTAINER" -) - -var ( - // inputs - driverType = flag.String(driverTypeArg, "", "task driver type, one of ROOT_DAG, DAG, CONTAINER") - pipelineName = flag.String("pipeline_name", "", "pipeline context name") - runID = flag.String("run_id", "", "pipeline run uid") - runName = flag.String("run_name", "", "pipeline run name (Kubernetes object name)") - runDisplayName = flag.String("run_display_name", "", "pipeline run display name") - componentSpecJson = flag.String("component", "{}", "component spec") - taskSpecJson = flag.String("task", "", "task spec") - runtimeConfigJson = flag.String("runtime_config", "", "jobruntime config") - iterationIndex = flag.Int("iteration_index", -1, "iteration index, -1 means not an interation") - taskName = flag.String("task_name", "", "original task name, used for proper input resolution in the container/dag driver") - - // container inputs - dagExecutionID = flag.Int64("dag_execution_id", 0, "DAG execution ID") - containerSpecJson = flag.String("container", "{}", "container spec") - k8sExecConfigJson = flag.String("kubernetes_config", "{}", "kubernetes executor config") - - // config - mlmdServerAddress = flag.String("mlmd_server_address", "", "MLMD server address") - mlmdServerPort = flag.String("mlmd_server_port", "", "MLMD server port") - - // output paths - executionIDPath = flag.String("execution_id_path", "", "Exeucution ID output path") - iterationCountPath = flag.String("iteration_count_path", "", "Iteration Count output path") - podSpecPatchPath = flag.String("pod_spec_patch_path", "", "Pod Spec Patch output path") - // the value stored in the paths will be either 'true' or 'false' - cachedDecisionPath = flag.String("cached_decision_path", "", "Cached Decision output path") - conditionPath = flag.String("condition_path", "", "Condition output path") - logLevel = flag.String("log_level", "1", "The verbosity level to log.") - - // proxy - httpProxy = flag.String(httpProxyArg, unsetProxyArgValue, "The proxy for HTTP connections.") - httpsProxy = flag.String(httpsProxyArg, unsetProxyArgValue, "The proxy for HTTPS connections.") - noProxy = flag.String(noProxyArg, unsetProxyArgValue, "Addresses that should ignore the proxy.") - publishLogs = flag.String("publish_logs", "true", "Whether to publish component logs to the object store") - cacheDisabledFlag = flag.Bool("cache_disabled", false, "Disable cache globally.") -) - -// func RootDAG(pipelineName string, runID string, component *pipelinespec.ComponentSpec, task *pipelinespec.PipelineTaskSpec, mlmd *metadata.Client) (*Execution, error) { - -func main() { - flag.Parse() - - glog.Infof("Setting log level to: '%s'", *logLevel) - err := flag.Set("v", *logLevel) - if err != nil { - glog.Warningf("Failed to set log level: %s", err.Error()) - } - - err = drive() - if err != nil { - glog.Exitf("%v", err) - } -} - -// Use WARNING default logging level to facilitate troubleshooting. -func init() { - flag.Set("logtostderr", "true") - // Change the WARNING to INFO level for debugging. - flag.Set("stderrthreshold", "WARNING") -} - -func validate() error { - if *driverType == "" { - return fmt.Errorf("argument --%s must be specified", driverTypeArg) - } - if *httpProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpProxyArg) - } - if *httpsProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpsProxyArg) - } - if *noProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", noProxyArg) - } - // validation responsibility lives in driver itself, so we do not validate all other args - return nil -} - -func drive() (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("KFP driver: %w", err) - } - }() - ctx := context.Background() - if err = validate(); err != nil { - return err - } - - proxy.InitializeConfig(*httpProxy, *httpsProxy, *noProxy) - - glog.Infof("input ComponentSpec:%s\n", prettyPrint(*componentSpecJson)) - componentSpec := &pipelinespec.ComponentSpec{} - if err := util.UnmarshalString(*componentSpecJson, componentSpec); err != nil { - return fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(*componentSpecJson)) - } - var taskSpec *pipelinespec.PipelineTaskSpec - if *taskSpecJson != "" { - glog.Infof("input TaskSpec:%s\n", prettyPrint(*taskSpecJson)) - taskSpec = &pipelinespec.PipelineTaskSpec{} - if err := util.UnmarshalString(*taskSpecJson, taskSpec); err != nil { - return fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, taskSpecJson) - } - } - glog.Infof("input ContainerSpec:%s\n", prettyPrint(*containerSpecJson)) - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} - if err := util.UnmarshalString(*containerSpecJson, containerSpec); err != nil { - return fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, containerSpecJson) - } - var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig - if *runtimeConfigJson != "" { - glog.Infof("input RuntimeConfig:%s\n", prettyPrint(*runtimeConfigJson)) - runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} - if err := util.UnmarshalString(*runtimeConfigJson, runtimeConfig); err != nil { - return fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, runtimeConfigJson) - } - } - k8sExecCfg, err := parseExecConfigJson(k8sExecConfigJson) - if err != nil { - return err - } - namespace, err := config.InPodNamespace() - if err != nil { - return err - } - client, err := newMlmdClient() - if err != nil { - return err - } - cacheClient, err := cacheutils.NewClient(*cacheDisabledFlag) - if err != nil { - return err - } - options := driver.Options{ - PipelineName: *pipelineName, - RunID: *runID, - RunName: *runName, - RunDisplayName: *runDisplayName, - Namespace: namespace, - Component: componentSpec, - Task: taskSpec, - DAGExecutionID: *dagExecutionID, - IterationIndex: *iterationIndex, - PipelineLogLevel: *logLevel, - PublishLogs: *publishLogs, - CacheDisabled: *cacheDisabledFlag, - DriverType: *driverType, - TaskName: *taskName, - } - var execution *driver.Execution - var driverErr error - switch *driverType { - case ROOT_DAG: - options.RuntimeConfig = runtimeConfig - execution, driverErr = driver.RootDAG(ctx, options, client) - case DAG: - execution, driverErr = driver.DAG(ctx, options, client) - case CONTAINER: - options.Container = containerSpec - options.KubernetesExecutorConfig = k8sExecCfg - execution, driverErr = driver.Container(ctx, options, client, cacheClient) - default: - err = fmt.Errorf("unknown driverType %s", *driverType) - } - if driverErr != nil { - if execution == nil { - return driverErr - } - defer func() { - // Override error with driver error, because driver error is more important. - // However, we continue running, because the following code prints debug info that - // may be helpful for figuring out why this failed. - err = driverErr - }() - } - - executionPaths := &ExecutionPaths{ - ExecutionID: *executionIDPath, - IterationCount: *iterationCountPath, - CachedDecision: *cachedDecisionPath, - Condition: *conditionPath, - PodSpecPatch: *podSpecPatchPath, - } - - return handleExecution(execution, *driverType, executionPaths) -} - -func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { - var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig - if *k8sExecConfigJson != "" { - glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) - k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} - if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { - return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) - } - } - return k8sExecCfg, nil -} - -func handleExecution(execution *driver.Execution, driverType string, executionPaths *ExecutionPaths) error { - if execution.ID != 0 { - glog.Infof("output execution.ID=%v", execution.ID) - if executionPaths.ExecutionID != "" { - if err := writeFile(executionPaths.ExecutionID, []byte(fmt.Sprint(execution.ID))); err != nil { - return fmt.Errorf("failed to write execution ID to file: %w", err) - } - } - } - if execution.IterationCount != nil { - if err := writeFile(executionPaths.IterationCount, []byte(fmt.Sprintf("%v", *execution.IterationCount))); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } else { - if driverType == ROOT_DAG { - if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } - } - if execution.Cached != nil { - if err := writeFile(executionPaths.CachedDecision, []byte(strconv.FormatBool(*execution.Cached))); err != nil { - return fmt.Errorf("failed to write cached decision to file: %w", err) - } - } - if execution.Condition != nil { - if err := writeFile(executionPaths.Condition, []byte(strconv.FormatBool(*execution.Condition))); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } else { - // nil is a valid value for Condition - if driverType == ROOT_DAG || driverType == CONTAINER { - if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } - } - if execution.PodSpecPatch != "" { - glog.Infof("output podSpecPatch=\n%s\n", execution.PodSpecPatch) - if executionPaths.PodSpecPatch == "" { - return fmt.Errorf("--pod_spec_patch_path is required for container executor drivers") - } - if err := writeFile(executionPaths.PodSpecPatch, []byte(execution.PodSpecPatch)); err != nil { - return fmt.Errorf("failed to write pod spec patch to file: %w", err) - } - } - if execution.ExecutorInput != nil { - executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) - if err != nil { - return fmt.Errorf("failed to marshal ExecutorInput to JSON: %w", err) - } - executorInputJSON := string(executorInputBytes) - glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) - } - return nil -} - -func prettyPrint(jsonStr string) string { - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") - if err != nil { - return jsonStr - } - return prettyJSON.String() -} - -func writeFile(path string, data []byte) (err error) { - if path == "" { - return fmt.Errorf("path is not specified") - } - defer func() { - if err != nil { - err = fmt.Errorf("failed to write to %s: %w", path, err) - } - }() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, data, 0o644) -} - -func newMlmdClient() (*metadata.Client, error) { - mlmdConfig := metadata.DefaultConfig() - if *mlmdServerAddress != "" && *mlmdServerPort != "" { - mlmdConfig.Address = *mlmdServerAddress - mlmdConfig.Port = *mlmdServerPort - } - return metadata.NewClient(mlmdConfig.Address, mlmdConfig.Port) -} diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 33edaf4b7f7..dc467a87723 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -443,7 +443,7 @@ var driverResources = k8score.ResourceRequirements{ // Launcher only copies the binary into the volume, so it needs minimal resources. var launcherResources = k8score.ResourceRequirements{ Limits: map[k8score.ResourceName]k8sres.Quantity{ - k8score.ResourceMemory: k8sres.MustParse("128Mi"), + k8score.ResourceMemory: k8sres.MustParse("256Mi"), k8score.ResourceCPU: k8sres.MustParse("0.5"), }, Requests: map[k8score.ResourceName]k8sres.Quantity{ diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index e03c60c6dae..b7301717e64 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -145,10 +145,15 @@ func GetPipelineRunAsUser() *int64 { return &runAsUser } -func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) { +func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs, error) { + template, err := c.addContainerDriverTemplate() + if err != nil { + return nil, nil, err + } + dagTask := &wfapi.DAGTask{ Name: name, - Template: c.addContainerDriverTemplate(), + Template: template, Arguments: wfapi.Arguments{ Parameters: []wfapi.Parameter{ {Name: paramComponent, Value: wfapi.AnyStringPtr(inputs.component)}, @@ -176,44 +181,44 @@ func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriv cached: taskOutputParameter(name, paramCachedDecision), condition: taskOutputParameter(name, paramCondition), } - return dagTask, outputs + return dagTask, outputs, nil } -func (c *workflowCompiler) addContainerDriverTemplate() string { +func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { name := "system-container-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", "CONTAINER", - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--container", inputValue(paramContainer), - "--iteration_index", inputValue(paramIterationIndex), - "--cached_decision_path", outputPath(paramCachedDecision), - "--pod_spec_patch_path", outputPath(paramPodSpecPatch), - "--condition_path", outputPath(paramCondition), - "--kubernetes_config", inputValue(paramKubernetesConfig), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) - } - if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + return name, nil + } + + logLevel, _ := os.LookupEnv(PipelineLogLevelEnvVar) + publishLogs, _ := os.LookupEnv(PublishLogsEnvVar) + + driverPlugin, err := driverPlugin(map[string]interface{}{ + "type": "CONTAINER", + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "container": inputValue(paramContainer), + "iteration_index": inputValue(paramIterationIndex), + "cached_decision_path": outputPath(paramCachedDecision), + "pod_spec_patch_path": outputPath(paramPodSpecPatch), + "condition_path": outputPath(paramCondition), + "kubernetes_config": inputValue(paramKubernetesConfig), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "cache_disabled": c.cacheDisabled, + "log_level": logLevel, + "publish_logs": publishLogs, + }) + if err != nil { + return "", fmt.Errorf("failed to create container driver template: %w", err) } t := &wfapi.Template{ @@ -231,22 +236,16 @@ func (c *workflowCompiler) addContainerDriverTemplate() string { }, Outputs: wfapi.Outputs{ Parameters: []wfapi.Parameter{ - {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, - {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/cached-decision", Default: wfapi.AnyStringPtr("false")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, + {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, + {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{JSONPath: "$.cached-decision", Default: wfapi.AnyStringPtr("false")}}, + {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.condition", Default: wfapi.AnyStringPtr("true")}}, }, }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: proxy.GetConfig().GetEnvVars(), - }, + Plugin: driverPlugin, } c.templates[name] = t c.wf.Spec.Templates = append(c.wf.Spec.Templates, *t) - return name + return name, nil } type containerExecutorInputs struct { diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index 27ad6f9b8b7..6b9bb26d11f 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -24,7 +24,6 @@ import ( wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/v2/compiler" - k8score "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -283,7 +282,7 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec driverTaskName := name + "-driver" // The following call will return an empty string for tasks without kubernetes-specific annotation. kubernetesConfigPlaceholder, _ := c.useKubernetesImpl(componentName) - driver, driverOutputs := c.containerDriverTask(driverTaskName, containerDriverInputs{ + driver, driverOutputs, err := c.containerDriverTask(driverTaskName, containerDriverInputs{ component: componentSpecPlaceholder, task: taskSpecJson, container: containerPlaceholder, @@ -292,6 +291,9 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec kubernetesConfig: kubernetesConfigPlaceholder, taskName: name, }) + if err != nil { + return nil, err + } if task.GetTriggerPolicy().GetCondition() == "" { driverOutputs.condition = "" } @@ -531,9 +533,13 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* Value: wfapi.AnyStringPtr(inputs.taskName), }) } + dagDriverTemplate, err := c.addDAGDriverTemplate() + if err != nil { + return nil, nil, err + } t := &wfapi.DAGTask{ Name: name, - Template: c.addDAGDriverTemplate(), + Template: dagDriverTemplate, Arguments: wfapi.Arguments{ Parameters: params, }, @@ -545,40 +551,40 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* }, nil } -func (c *workflowCompiler) addDAGDriverTemplate() string { +func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { name := "system-dag-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", inputValue(paramDriverType), - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--runtime_config", inputValue(paramRuntimeConfig), - "--iteration_index", inputValue(paramIterationIndex), - "--execution_id_path", outputPath(paramExecutionID), - "--iteration_count_path", outputPath(paramIterationCount), - "--condition_path", outputPath(paramCondition), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) - } - if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + return name, nil + } + + logLevel, _ := os.LookupEnv(PipelineLogLevelEnvVar) + publishLogs, _ := os.LookupEnv(PublishLogsEnvVar) + + driverPlugin, err := driverPlugin(map[string]interface{}{ + "type": inputValue(paramDriverType), + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "runtime_config": inputValue(paramRuntimeConfig), + "iteration_index": inputValue(paramIterationIndex), + "execution_id_path": outputPath(paramExecutionID), + "iteration_count_path": outputPath(paramIterationCount), + "condition_path": outputPath(paramCondition), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "cache_disabled": c.cacheDisabled, + "log_level": logLevel, + "publish_logs": publishLogs, + }) + if err != nil { + return "", fmt.Errorf("failed to create dag driver template: %w", err) } t := &wfapi.Template{ @@ -596,22 +602,16 @@ func (c *workflowCompiler) addDAGDriverTemplate() string { }, Outputs: wfapi.Outputs{ Parameters: []wfapi.Parameter{ - {Name: paramExecutionID, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/execution-id"}}, - {Name: paramIterationCount, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/iteration-count", Default: wfapi.AnyStringPtr("0")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, + {Name: paramExecutionID, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.execution-id"}}, + {Name: paramIterationCount, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.iteration-count", Default: wfapi.AnyStringPtr("0")}}, + {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.condition", Default: wfapi.AnyStringPtr("true")}}, }, }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: proxy.GetConfig().GetEnvVars(), - }, + Plugin: driverPlugin, } c.templates[name] = t c.wf.Spec.Templates = append(c.wf.Spec.Templates, *t) - return name + return name, nil } func addImplicitDependencies(dagSpec *pipelinespec.DagSpec) error { diff --git a/backend/src/v2/compiler/argocompiler/plugin.go b/backend/src/v2/compiler/argocompiler/plugin.go new file mode 100644 index 00000000000..941f9fcd04b --- /dev/null +++ b/backend/src/v2/compiler/argocompiler/plugin.go @@ -0,0 +1,22 @@ +package argocompiler + +import ( + "encoding/json" + "fmt" + wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +func driverPlugin(params map[string]interface{}) (*wfapi.Plugin, error) { + pluginConfig := map[string]interface{}{ + "driver-plugin": map[string]interface{}{ + "args": params, + }, + } + jsonConfig, err := json.Marshal(pluginConfig) + if err != nil { + return nil, fmt.Errorf("driver plugin creation error: marshaling plugin config to JSON failed: %w", err) + } + return &wfapi.Plugin{Object: wfapi.Object{ + Value: jsonConfig, + }}, nil +} diff --git a/manifests/kustomize/base/pipeline/kustomization.yaml b/manifests/kustomize/base/pipeline/kustomization.yaml index e17b443a0d5..dbb90e1fb87 100644 --- a/manifests/kustomize/base/pipeline/kustomization.yaml +++ b/manifests/kustomize/base/pipeline/kustomization.yaml @@ -35,6 +35,7 @@ resources: - viewer-sa.yaml - kfp-launcher-configmap.yaml - allow-same-namespace-networkpolicy.yaml + - ml-pipeline-driver-plugin-cm.yaml images: - name: ghcr.io/kubeflow/kfp-api-server newTag: 2.14.3 diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index cd9f3dd2962..d0a7363b1f5 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -126,7 +126,7 @@ spec: value: ghcr.io/kubeflow/kfp-driver:2.14.3 - name: V2_LAUNCHER_IMAGE value: ghcr.io/kubeflow/kfp-launcher:2.14.3 - image: ghcr.io/kubeflow/kfp-api-server:dummy + image: ntny/kfp-api-server:central-driver-poc imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml new file mode 100644 index 00000000000..da703b2c66d --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + workflows.argoproj.io/configmap-type: ExecutorPlugin + name: ml-pipeline-driver-agent +data: + sidecar.automountServiceAccountToken: "true" + sidecar.container: | + image: ntny/kfp-driver:central-driver-poc + name: driver-plugin + ports: + - containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index eba0ee9f2d6..8eb2f07beef 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -33,15 +33,19 @@ rules: - delete - get - apiGroups: - - argoproj.io + - argoproj.io resources: - workflows + - workflowtaskresults + - workflowtasksets + - "workflowtasksets/status" verbs: - get - - list + - create + - patch - watch + - list - update - - patch - apiGroups: - "" resources: @@ -85,3 +89,44 @@ rules: verbs: - create - patch + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ml-pipeline-driver-agent-executor-plugin + labels: + application-crd-id: kubeflow-pipelines +secrets: + - name: ml-pipeline-driver-agent-executor-plugin.service-account-token + +--- +apiVersion: v1 +kind: Secret +metadata: + name: ml-pipeline-driver-agent-executor-plugin.service-account-token + annotations: + kubernetes.io/service-account.name: ml-pipeline-driver-agent-executor-plugin +type: kubernetes.io/service-account-token + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: configmap-reader +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] # права на чтение +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: configmap-reader-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline-driver-agent-executor-plugin +roleRef: + kind: Role + name: configmap-reader + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml index 8cb2c669fb2..e1f42088371 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml @@ -2,3 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: name: pipeline-runner + +--- +apiVersion: v1 +kind: Secret +metadata: + name: pipeline-runner.service-account-token + annotations: + kubernetes.io/service-account.name: pipeline-runner +type: kubernetes.io/service-account-token diff --git a/manifests/kustomize/env/dev/kustomization.yaml b/manifests/kustomize/env/dev/kustomization.yaml index f2ed5ddabf1..72b7adbbf33 100644 --- a/manifests/kustomize/env/dev/kustomization.yaml +++ b/manifests/kustomize/env/dev/kustomization.yaml @@ -9,8 +9,8 @@ resources: - ../gcp/inverse-proxy images: -- name: ghcr.io/kubeflow/kfp-api-server - newTag: master +- name: ntny/kfp-driver + newTag: central-driver-poc - name: ghcr.io/kubeflow/kfp-frontend newTag: master - name: ghcr.io/kubeflow/kfp-persistence-agent diff --git a/manifests/kustomize/third-party/application/application-controller-deployment.yaml b/manifests/kustomize/third-party/application/application-controller-deployment.yaml index 1f1c589aaef..2f4277da116 100644 --- a/manifests/kustomize/third-party/application/application-controller-deployment.yaml +++ b/manifests/kustomize/third-party/application/application-controller-deployment.yaml @@ -31,8 +31,8 @@ spec: resources: limits: cpu: 100m - memory: 30Mi + memory: 128Mi requests: cpu: 100m - memory: 20Mi + memory: 64Mi serviceAccountName: application diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml index 2ea35587b7b..db3326c255e 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml @@ -16,6 +16,9 @@ spec: - workflow-controller-configmap - --executor-image - quay.io/argoproj/argoexec:v3.6.7 + env: + - name: ARGO_EXECUTOR_PLUGINS + value: "true" securityContext: readOnlyRootFilesystem: true runAsNonRoot: true diff --git a/test_data/compiled-workflows/arguments-parameters.yaml b/test_data/compiled-workflows/arguments-parameters.yaml index 8a6ae6faa98..3e4bce71654 100644 --- a/test_data/compiled-workflows/arguments-parameters.yaml +++ b/test_data/compiled-workflows/arguments-parameters.yaml @@ -20,307 +20,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}' - name: container - value: '{{workflow.parameters.implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task-name - value: echo - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' - depends: echo-driver.Succeeded - name: echo - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"param1":"hello"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + depends: hello-world-driver.Succeeded + name: hello-world + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/arguments.pipeline.yaml b/test_data/compiled-workflows/arguments.pipeline.yaml index 8a6ae6faa98..0de1a8f645b 100644 --- a/test_data/compiled-workflows/arguments.pipeline.yaml +++ b/test_data/compiled-workflows/arguments.pipeline.yaml @@ -6,12 +6,17 @@ metadata: spec: arguments: parameters: - - name: components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 - value: '{"executorLabel":"exec-echo","inputDefinitions":{"parameters":{"param1":{"parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' - - name: implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 - value: '{"args":["{{$.inputs.parameters[''param1'']}}-{{$.inputs.parameters[''param2'']}}"],"command":["echo"],"image":"public.ecr.aws/docker/library/python:3.12"}' + - name: components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7 + value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + - name: implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7 + value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf + \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def + hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser + = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", + dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args + = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.9"}' - name: components-root - value: '{"dag":{"tasks":{"echo":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}}},"inputDefinitions":{"parameters":{"param1":{"defaultValue":"hello","parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' + value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -20,307 +25,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}' - name: container - value: '{{workflow.parameters.implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task-name - value: echo - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' - depends: echo-driver.Succeeded - name: echo - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"param1":"hello"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + depends: hello-world-driver.Succeeded + name: hello-world + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index 291c3066967..21f3a0a3f7b 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -48,390 +48,353 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-1b448d8700f1af7ecebf0d5b47c96c95fd831c03586af16ac2fb3e7478f081e1}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-1b448d8700f1af7ecebf0d5b47c96c95fd831c03586af16ac2fb3e7478f081e1}}' - name: task-name - value: core-comp - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: core-comp-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' - depends: core-comp-driver.Succeeded - name: core-comp - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-core - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-core}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core"},"taskInfo":{"name":"core"}}' - - name: task-name - value: core - name: core-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + valueFrom: + default: "false" + jsonPath: $.cached-decision - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' - depends: core-driver.Succeeded - name: core - template: comp-core - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-mantle - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-eeb59a2dbac5373b0713b02eabada5ee95398f70f8a1980755b51b8578844ab1}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"artifacts":{"input":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - - name: container - value: '{{workflow.parameters.implementations-eeb59a2dbac5373b0713b02eabada5ee95398f70f8a1980755b51b8578844ab1}}' - - name: task-name - value: crust-comp - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: mantle.Succeeded - name: crust-comp-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + - name: retry-max-count + value: '{{inputs.parameters.retry-max-count}}' + - name: retry-backoff-duration + value: '{{inputs.parameters.retry-backoff-duration}}' + - name: retry-backoff-factor + value: '{{inputs.parameters.retry-backoff-factor}}' + - name: retry-backoff-max-duration + value: '{{inputs.parameters.retry-backoff-max-duration}}' + name: executor + template: retry-system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' - depends: crust-comp-driver.Succeeded - name: crust-comp - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-mantle}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-mantle"},"taskInfo":{"name":"mantle"}}' - - name: task-name - value: mantle - name: mantle-driver - template: system-dag-driver - - arguments: - parameters: + - default: "0" + name: retry-max-count + - default: "0" + name: retry-backoff-duration + - default: "0" + name: retry-backoff-factor + - default: "0" + name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + - name: retry-max-count + - name: retry-backoff-duration + - name: retry-backoff-factor + - name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + retryStrategy: + backoff: + duration: '{{inputs.parameters.retry-backoff-duration}}' + factor: '{{inputs.parameters.retry-backoff-factor}}' + maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' + limit: '{{inputs.parameters.retry-max-count}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-a"}}' + - name: container + value: '{{workflow.parameters.implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' + - name: task-name + value: component-a + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-a-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.component-a-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.component-a-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "2" + - name: retry-backoff-duration + value: "0" + - name: retry-backoff-factor + value: "2" + - name: retry-backoff-max-duration + value: "3600" + depends: component-a-driver.Succeeded + name: component-a + template: retry-system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}' + - name: container + value: '{{workflow.parameters.implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' + - name: task-name + value: component-b + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-b-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.component-b-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.component-b-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "2" + - name: retry-backoff-duration + value: "0" + - name: retry-backoff-factor + value: "2" + - name: retry-backoff-max-duration + value: "3600" + depends: component-b-driver.Succeeded + name: component-b + template: retry-system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' - depends: mantle-driver.Succeeded - name: mantle - template: comp-mantle - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: + metadata: {} + name: comp-nested-pipeline + outputs: {} + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-nested-pipeline}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"nested-pipeline"}}' + - name: task-name + value: nested-pipeline + name: nested-pipeline-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' + depends: nested-pipeline-driver.Succeeded + name: nested-pipeline + template: comp-nested-pipeline + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index 5ee2195e24f..9716360989f 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -20,307 +20,259 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world-container"},"taskInfo":{"name":"hello-world-container"}}' - name: container - value: '{{workflow.parameters.implementations-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' - name: task-name - value: hello-world-container - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-container-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.hello-world-container-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.hello-world-container-driver.outputs.parameters.cached-decision}}' - depends: hello-world-container-driver.Succeeded - name: hello-world-container - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + - --cache_disabled + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + depends: hello-world-driver.Succeeded + name: hello-world + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index 5cc4408ad8c..ca0074d5958 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -20,307 +20,260 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-io"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"container-io"}}' - name: container - value: '{{workflow.parameters.implementations-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105}}' - name: task-name - value: container-io - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: container-io-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.container-io-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.container-io-driver.outputs.parameters.cached-decision}}' - depends: container-io-driver.Succeeded - name: container-io - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "3" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + - --log_level + - "3" + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + depends: hello-world-driver.Succeeded + name: hello-world + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "3" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 2182a0adea8..ba7c590ff43 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -29,307 +29,270 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-ad34ccfb38dd37789911de71149bcf7f9c413a59602a851919d36e4da8d5a0c5}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dict-input"},"inputs":{"parameters":{"struct":{"componentInputParameter":"struct"}}},"taskInfo":{"name":"dict-input"}}' - name: container - value: '{{workflow.parameters.implementations-ad34ccfb38dd37789911de71149bcf7f9c413a59602a851919d36e4da8d5a0c5}}' - name: task-name - value: dict-input - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: dict-input-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.dict-input-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.dict-input-driver.outputs.parameters.cached-decision}}' - depends: dict-input-driver.Succeeded - name: dict-input - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + depends: hello-world-driver.Succeeded + name: hello-world + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} + volumeClaimTemplates: + - metadata: + creationTimestamp: null + name: kfp-workspace + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 250Gi + storageClassName: super-fast-storage + status: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/importer_pipeline_compiled.yaml b/test_data/compiled-workflows/importer_pipeline_compiled.yaml index cc37b49f9b3..8204faf7986 100644 --- a/test_data/compiled-workflows/importer_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/importer_pipeline_compiled.yaml @@ -41,377 +41,406 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --executor_type - - importer - - --task_spec - - '{{inputs.parameters.task}}' - - --component_spec - - '{{inputs.parameters.component}}' - - --importer_spec - - '{{inputs.parameters.importer}}' - - --pipeline_name - - my-pipe - - --run_id - - '{{workflow.uid}}' - - --parent_dag_id - - '{{inputs.parameters.parent-dag-id}}' - - --pod_name - - $(KFP_POD_NAME) - - --pod_uid - - $(KFP_POD_UID) - - --mlmd_server_address - - $(METADATA_GRPC_SERVICE_HOST) - - --mlmd_server_port - - $(METADATA_GRPC_SERVICE_PORT) - command: - - launcher-v2 - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: ghcr.io/kubeflow/kfp-launcher:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: task - - name: component - - name: importer - - name: parent-dag-id - metadata: {} - name: system-importer - outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipe - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"componentInputParameter":"artifact_uri"}}},"taskInfo":{"name":"importer"}}' - - name: component - value: '{{workflow.parameters.components-comp-importer}}' - - name: importer - value: '{{workflow.parameters.implementations-comp-importer}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: importer - template: system-importer - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-35e72db0aa0c9f3f658bdf4106261026ae722e5b716413f691cda4db8679a44e}}' - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-normalize-dataset"},"dependentTasks":["importer"],"inputs":{"artifacts":{"input_iris_dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}},"parameters":{"standard_scaler":{"componentInputParameter":"standard_scaler"}}},"taskInfo":{"name":"normalize-dataset"}}' - name: container - value: '{{workflow.parameters.implementations-35e72db0aa0c9f3f658bdf4106261026ae722e5b716413f691cda4db8679a44e}}' - name: task-name - value: normalize-dataset - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: importer.Succeeded - name: normalize-dataset-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.normalize-dataset-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.normalize-dataset-driver.outputs.parameters.cached-decision}}' - depends: normalize-dataset-driver.Succeeded - name: normalize-dataset - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipe - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"standard_scaler":true}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + - name: retry-max-count + value: '{{inputs.parameters.retry-max-count}}' + - name: retry-backoff-duration + value: '{{inputs.parameters.retry-backoff-duration}}' + - name: retry-backoff-factor + value: '{{inputs.parameters.retry-backoff-factor}}' + - name: retry-backoff-max-duration + value: '{{inputs.parameters.retry-backoff-max-duration}}' + name: executor + template: retry-system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + - default: "0" + name: retry-max-count + - default: "0" + name: retry-backoff-duration + - default: "0" + name: retry-backoff-factor + - default: "0" + name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + - name: retry-max-count + - name: retry-backoff-duration + - name: retry-backoff-factor + - name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + retryStrategy: + backoff: + duration: '{{inputs.parameters.retry-backoff-duration}}' + factor: '{{inputs.parameters.retry-backoff-factor}}' + maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' + limit: '{{inputs.parameters.retry-max-count}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffDuration":"1s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "2" + - name: retry-backoff-duration + value: "1" + - name: retry-backoff-factor + value: "2" + - name: retry-backoff-max-duration + value: "3600" + depends: hello-world-driver.Succeeded + name: hello-world + template: retry-system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world-2"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world-non-retry + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-non-retry-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.cached-decision}}' + depends: hello-world-non-retry-driver.Succeeded + name: hello-world-non-retry + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: namespace/n1/pipeline/hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index 710aa68bff1..e7438609e12 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -10,6 +10,10 @@ spec: value: '{"executorLabel":"exec-component-bool","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"BOOLEAN"}}}}' - name: implementations-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_bool"],"command":["sh","-c","\nif + - name: components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603 + value: '{"executorLabel":"exec-component-a"}' + - name: implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603 + value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_a"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.2'' ''--no-deps'' @@ -24,6 +28,11 @@ spec: value: '{"executorLabel":"exec-component-int","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - name: implementations-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_int"],"command":["sh","-c","\nif + component_a():\n print(''Component A'')\n\n"],"image":"python:3.9"}' + - name: components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23 + value: '{"executorLabel":"exec-component-b"}' + - name: implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23 + value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_b"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.2'' ''--no-deps'' @@ -52,6 +61,11 @@ spec: value: '{"dag":{"tasks":{"component-bool":{"cachingOptions":{},"componentRef":{"name":"comp-component-bool"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool"}},"component-int":{"cachingOptions":{},"componentRef":{"name":"comp-component-int"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int"}},"component-str":{"cachingOptions":{},"componentRef":{"name":"comp-component-str"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str"}}}},"inputDefinitions":{"parameters":{"nestedInputBool":{"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputInt":{"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputStr":{"isOptional":true,"parameterType":"STRING"}}}}' - name: components-root value: '{"dag":{"tasks":{"nested-pipeline":{"cachingOptions":{},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}}}}' + component_b():\n print (''Component B'')\n\n"],"image":"python:3.9"}' + - name: components-comp-nested-pipeline + value: '{"dag":{"tasks":{"component-a":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"taskInfo":{"name":"component-a"}},"component-b":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}}}}' + - name: components-root + value: '{"dag":{"tasks":{"nested-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -60,384 +74,435 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce}}' - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-bool"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool"}}' - name: container - value: '{{workflow.parameters.implementations-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce}}' - name: task-name - value: component-bool - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-bool-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.component-bool-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.component-bool-driver.outputs.parameters.cached-decision}}' - depends: component-bool-driver.Succeeded - name: component-bool - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-int"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int"}}' - - name: container - value: '{{workflow.parameters.implementations-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934}}' - - name: task-name - value: component-int - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-int-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.component-int-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.component-int-driver.outputs.parameters.cached-decision}}' - depends: component-int-driver.Succeeded - name: component-int - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-str"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str"}}' - - name: container - value: '{{workflow.parameters.implementations-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2}}' - - name: task-name - value: component-str - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-str-driver - template: system-container-driver - - arguments: - parameters: + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + - name: retry-max-count + value: '{{inputs.parameters.retry-max-count}}' + - name: retry-backoff-duration + value: '{{inputs.parameters.retry-backoff-duration}}' + - name: retry-backoff-factor + value: '{{inputs.parameters.retry-backoff-factor}}' + - name: retry-backoff-max-duration + value: '{{inputs.parameters.retry-backoff-max-duration}}' + name: executor + template: retry-system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.component-str-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.component-str-driver.outputs.parameters.cached-decision}}' - depends: component-str-driver.Succeeded - name: component-str - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-nested-pipeline - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-nested-pipeline}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}' - - name: task-name - value: nested-pipeline - name: nested-pipeline-driver - template: system-dag-driver - - arguments: - parameters: + - default: "0" + name: retry-max-count + - default: "0" + name: retry-backoff-duration + - default: "0" + name: retry-backoff-factor + - default: "0" + name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + - name: retry-max-count + - name: retry-backoff-duration + - name: retry-backoff-factor + - name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + retryStrategy: + backoff: + duration: '{{inputs.parameters.retry-backoff-duration}}' + factor: '{{inputs.parameters.retry-backoff-factor}}' + maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' + limit: '{{inputs.parameters.retry-max-count}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"taskInfo":{"name":"component-a"}}' + - name: container + value: '{{workflow.parameters.implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' + - name: task-name + value: component-a + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-a-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.component-a-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.component-a-driver.outputs.parameters.cached-decision}}' + depends: component-a-driver.Succeeded + name: component-a + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}' + - name: container + value: '{{workflow.parameters.implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' + - name: task-name + value: component-b + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-b-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.component-b-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.component-b-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "2" + - name: retry-backoff-duration + value: "0" + - name: retry-backoff-factor + value: "2" + - name: retry-backoff-max-duration + value: "3600" + depends: component-b-driver.Succeeded + name: component-b + template: retry-system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' - depends: nested-pipeline-driver.Succeeded - name: nested-pipeline - template: comp-nested-pipeline - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: + metadata: {} + name: comp-nested-pipeline + outputs: {} + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-nested-pipeline}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}' + - name: task-name + value: nested-pipeline + name: nested-pipeline-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' + depends: nested-pipeline-driver.Succeeded + name: nested-pipeline + template: comp-nested-pipeline + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 28faf0eeb68..854a8f8e61c 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -46,390 +46,353 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-9c2adbce22e7223ac5155c235fbc1541b292d9e1f78a984687c391207f9bdbce}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container - value: '{{workflow.parameters.implementations-9c2adbce22e7223ac5155c235fbc1541b292d9e1f78a984687c391207f9bdbce}}' - name: task-name - value: core-comp - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: core-comp-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' - depends: core-comp-driver.Succeeded - name: core-comp - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-core - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-core}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core"},"taskInfo":{"name":"core"}}' - - name: task-name - value: core - name: core-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.core-driver.outputs.parameters.execution-id}}' + valueFrom: + default: "false" + jsonPath: $.cached-decision - name: condition - value: '{{tasks.core-driver.outputs.parameters.condition}}' - depends: core-driver.Succeeded - name: core - template: comp-core - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-mantle - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c2d514fad77a56dbbca5ace4aea88a97d5d0872463870c829ae73b6e7c7706b3}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"parameters":{"input":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' - - name: container - value: '{{workflow.parameters.implementations-c2d514fad77a56dbbca5ace4aea88a97d5d0872463870c829ae73b6e7c7706b3}}' - - name: task-name - value: crust-comp - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: mantle.Succeeded - name: crust-comp-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: hello-world + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + - name: retry-max-count + value: '{{inputs.parameters.retry-max-count}}' + - name: retry-backoff-duration + value: '{{inputs.parameters.retry-backoff-duration}}' + - name: retry-backoff-factor + value: '{{inputs.parameters.retry-backoff-factor}}' + - name: retry-backoff-max-duration + value: '{{inputs.parameters.retry-backoff-max-duration}}' + name: executor + template: retry-system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' - depends: crust-comp-driver.Succeeded - name: crust-comp - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-mantle}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-mantle"},"taskInfo":{"name":"mantle"}}' - - name: task-name - value: mantle - name: mantle-driver - template: system-dag-driver - - arguments: - parameters: + - default: "0" + name: retry-max-count + - default: "0" + name: retry-backoff-duration + - default: "0" + name: retry-backoff-factor + - default: "0" + name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + - name: retry-max-count + - name: retry-backoff-duration + - name: retry-backoff-factor + - name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + retryStrategy: + backoff: + duration: '{{inputs.parameters.retry-backoff-duration}}' + factor: '{{inputs.parameters.retry-backoff-factor}}' + maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' + limit: '{{inputs.parameters.retry-max-count}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":1,"backoffMaxDuration":"1800s","maxRetryCount":1},"taskInfo":{"name":"component-a"}}' + - name: container + value: '{{workflow.parameters.implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' + - name: task-name + value: component-a + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-a-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.component-a-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.component-a-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "1" + - name: retry-backoff-duration + value: "0" + - name: retry-backoff-factor + value: "1" + - name: retry-backoff-max-duration + value: "1800" + depends: component-a-driver.Succeeded + name: component-a + template: retry-system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}' + - name: container + value: '{{workflow.parameters.implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' + - name: task-name + value: component-b + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-b-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.component-b-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.component-b-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "2" + - name: retry-backoff-duration + value: "0" + - name: retry-backoff-factor + value: "2" + - name: retry-backoff-max-duration + value: "3600" + depends: component-b-driver.Succeeded + name: component-b + template: retry-system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.mantle-driver.outputs.parameters.condition}}' - depends: mantle-driver.Succeeded - name: mantle - template: comp-mantle - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: + metadata: {} + name: comp-nested-pipeline + outputs: {} + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: hello-world + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-nested-pipeline}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":1,"backoffMaxDuration":"1800s","maxRetryCount":1},"taskInfo":{"name":"nested-pipeline"}}' + - name: task-name + value: nested-pipeline + name: nested-pipeline-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' + depends: nested-pipeline-driver.Succeeded + name: nested-pipeline + template: comp-nested-pipeline + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index 6ad43a7c158..ba53f4f0c61 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -44,547 +44,498 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op2"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"world"}}}},"taskInfo":{"name":"print-op2"}}' - name: container - value: '{{workflow.parameters.implementations-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' - name: task-name - value: print-op2 - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op2-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' - depends: print-op2-driver.Succeeded - name: print-op2 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-condition-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"Bye!"}}}},"taskInfo":{"name":"print-op"}}' - - name: container - value: '{{workflow.parameters.implementations-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' - - name: task-name - value: print-op - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' - depends: print-op-driver.Succeeded - name: print-op - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-condition-2 - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-condition-1}}' + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op"}}' + - name: container + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task-name + value: print-op + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + depends: print-op-driver.Succeeded + name: print-op + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-2"}}' + - name: container + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task-name + value: print-op-2 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-2-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + depends: print-op-2-driver.Succeeded + name: print-op-2 + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-1"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"pipelinechannel--print-op1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--print-op1-Output''] - == ''Hello''"}}' - - name: task-name - value: condition-1 - depends: print-op1.Succeeded - name: condition-1-driver - template: system-dag-driver - - arguments: - parameters: + metadata: {} + name: comp-for-loop-2 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op-3"}}' + - name: container + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task-name + value: print-op-3 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-3-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' + depends: print-op-3-driver.Succeeded + name: print-op-3 + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-4"}}' + - name: container + value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' + - name: task-name + value: print-op-4 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-4-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op-4-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-4-driver.outputs.parameters.cached-decision}}' + depends: print-op-4-driver.Succeeded + name: print-op-4 + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' - depends: condition-1-driver.Succeeded - name: condition-1 - template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: + metadata: {} + name: comp-for-loop-4 + outputs: {} + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-comp-condition-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-2"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"pipelinechannel--print-op1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--print-op1-Output''] - != ''Hello''"}}' - - name: task-name - value: condition-2 - depends: print-op1.Succeeded - name: condition-2-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' - depends: condition-2-driver.Succeeded - name: condition-2 - template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - - name: container - value: '{{workflow.parameters.implementations-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' - - name: task-name - value: print-op1 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op1-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' - depends: print-op1-driver.Succeeded - name: print-op1 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-inner-pipeline - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-inner-pipeline}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-inner-pipeline"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"inner-pipeline"}}' - - name: task-name - value: inner-pipeline - name: inner-pipeline-driver - template: system-dag-driver - - arguments: - parameters: + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-for-loop-2}}' + - name: iteration-index + value: '{{inputs.parameters.iteration-index}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-for-loop-2"},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"A_a\": + \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": + \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": + \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": + \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' + name: iteration-item-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + depends: iteration-item-driver.Succeeded + name: iteration-item + template: comp-for-loop-2 + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' - depends: inner-pipeline-driver.Succeeded - name: inner-pipeline - template: comp-inner-pipeline - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"Hello\", - \"world!\"]"}},"taskInfo":{"name":"for-loop-2"}}' - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: + metadata: {} + name: comp-for-loop-2-iteration + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-for-loop-2}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-for-loop-2"},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"A_a\": + \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": + \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": + \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": + \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' + name: iteration-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + - name: iteration-index + value: '{{item}}' + depends: iteration-driver.Succeeded + name: iteration-iterations + template: comp-for-loop-2-iteration + withSequence: + count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-2 - inputs: - parameters: - - name: parent-dag-id - - name: iteration-index - metadata: {} - name: comp-for-loop-2-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' + metadata: {} + name: comp-for-loop-2-for-loop-2-iterator + outputs: {} + parallelism: 2 + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-for-loop-4}}' + - name: iteration-index + value: '{{inputs.parameters.iteration-index}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-for-loop-4"},"iteratorPolicy":{"parallelismLimit":4},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": + \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": + \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": + \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": + \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' + name: iteration-item-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + depends: iteration-item-driver.Succeeded + name: iteration-item + template: comp-for-loop-4 + inputs: + parameters: - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"Hello\", - \"world!\"]"}},"taskInfo":{"name":"for-loop-2"}}' - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-2-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-for-loop-2-for-loop-2-iterator - outputs: {} - - dag: - tasks: - - arguments: - parameters: + metadata: {} + name: comp-for-loop-4-iteration + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-for-loop-4}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-for-loop-4"},"iteratorPolicy":{"parallelismLimit":4},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": + \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": + \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": + \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": + \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": + \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' + name: iteration-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' + - name: iteration-index + value: '{{item}}' + depends: iteration-driver.Succeeded + name: iteration-iterations + template: comp-for-loop-4-iteration + withSequence: + count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + inputs: + parameters: - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-2 - template: comp-for-loop-2-for-loop-2-iterator - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' - - name: container - value: '{{workflow.parameters.implementations-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' - - name: task-name - value: print-op1 + metadata: {} + name: comp-for-loop-4-for-loop-4-iterator + outputs: {} + parallelism: 4 + - dag: + tasks: + - arguments: + parameters: + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: for-loop-2 + template: comp-for-loop-2-for-loop-2-iterator + - arguments: + parameters: + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: for-loop-4 + template: comp-for-loop-4-for-loop-4-iterator + inputs: + parameters: - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op1-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' - depends: print-op1-driver.Succeeded - name: print-op1 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"msg":"Hello"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"text":{"stringValue":"hello world"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index 9cb8c51019c..e0784f8f808 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -47,401 +47,352 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-73aa41a182947039e1bae4bb36fe13a58b2239d2987abbba8f4a51fcc3938390}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task - failed."}}}},"taskInfo":{"name":"fail-op"}}' - name: container - value: '{{workflow.parameters.implementations-73aa41a182947039e1bae4bb36fe13a58b2239d2987abbba8f4a51fcc3938390}}' - name: task-name - value: fail-op - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: fail-op-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' - depends: fail-op-driver.Succeeded - name: fail-op - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}' - - name: container - value: '{{workflow.parameters.implementations-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' - - name: task-name - value: print-op-2 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-2-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' - depends: print-op-2-driver.Succeeded - name: print-op-2 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-exit-handler-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit - handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - - name: container - value: '{{workflow.parameters.implementations-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' - - name: task-name - value: print-op - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-driver - template: system-container-driver - - arguments: - parameters: + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' - depends: print-op-driver.Succeeded - name: print-op - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: exit-hook-root-print-op - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-exit-handler-1}}' + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-8444a6bac7a3d81cc54291a13166a74231d98f9a98861815f15a055edde30ed8}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constantValue":{"stringValue":"Task + failed."}}}}},"taskInfo":{"name":"fail-op"}}' + - name: container + value: '{{workflow.parameters.implementations-8444a6bac7a3d81cc54291a13166a74231d98f9a98861815f15a055edde30ed8}}' + - name: task-name + value: fail-op + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: fail-op-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + depends: fail-op-driver.Succeeded + name: fail-op + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelineparam--message"}}},"taskInfo":{"name":"print-op-2"}}' + - name: container + value: '{{workflow.parameters.implementations-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' + - name: task-name + value: print-op-2 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-2-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + depends: print-op-2-driver.Succeeded + name: print-op-2 + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}}' - - name: task-name - value: exit-handler-1 - name: exit-handler-1-driver - template: system-dag-driver - - arguments: - parameters: + metadata: {} + name: comp-exit-handler-1 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' + - name: task + value: '{"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constantValue":{"stringValue":"Exit + handler has worked!"}}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' + - name: container + value: '{{workflow.parameters.implementations-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' + - name: task-name + value: print-op + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + depends: print-op-driver.Succeeded + name: print-op + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + metadata: {} + name: exit-hook-root-print-op + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' - depends: exit-handler-1-driver.Succeeded - hooks: - exit: - arguments: + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - template: exit-hook-root-print-op - name: exit-handler-1 - template: comp-exit-handler-1 - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"message":"Hello World!"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + - name: component + value: '{{workflow.parameters.components-comp-exit-handler-1}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelineparam--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}}' + - name: task-name + value: exit-handler-1 + name: exit-handler-1-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + depends: exit-handler-1-driver.Succeeded + hooks: + exit: + arguments: + parameters: + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + template: exit-hook-root-print-op + name: exit-handler-1 + template: comp-exit-handler-1 + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameters":{"message":{"stringValue":"Hello World!"}}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index 0096adb03c9..e5a2a1749b3 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -10,6 +10,10 @@ spec: value: '{"executorLabel":"exec-echo-state","inputDefinitions":{"parameters":{"status":{"isOptional":true,"parameterType":"TASK_FINAL_STATUS"}}}}' - name: implementations-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo_state"],"command":["sh","-c","\nif + - name: components-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618 + value: '{"executorLabel":"exec-echo-state","inputDefinitions":{"parameters":{"status":{"parameterType":"TASK_FINAL_STATUS"}}}}' + - name: implementations-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618 + value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo_state"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' @@ -26,6 +30,11 @@ spec: value: '{"executorLabel":"exec-some-task"}' - name: implementations-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566 value: '{"args":["--executor_input","{{$}}","--function_to_execute","some_task"],"command":["sh","-c","\nif + echo_state(status: str):\n print(status)\n\n"],"image":"python:3.9"}' + - name: components-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7 + value: '{"executorLabel":"exec-error"}' + - name: implementations-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7 + value: '{"args":["--executor_input","{{$}}","--function_to_execute","error"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' @@ -39,6 +48,11 @@ spec: value: '{"dag":{"tasks":{"some-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-some-task"},"taskInfo":{"name":"some-task"}}}}}' - name: components-root value: '{"dag":{"tasks":{"echo-state":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}},"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}}}}' + error():\n raise RuntimeError(\"An error\")\n\n"],"image":"python:3.9"}' + - name: components-comp-exit-handler-1 + value: '{"dag":{"tasks":{"error":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-error"},"taskInfo":{"name":"error"}}}}}' + - name: components-root + value: '{"dag":{"tasks":{"echo-state":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}},"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -47,375 +61,326 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-some-task"},"taskInfo":{"name":"some-task"}}' - name: container - value: '{{workflow.parameters.implementations-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566}}' - name: task-name - value: some-task - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: some-task-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.some-task-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.some-task-driver.outputs.parameters.cached-decision}}' - depends: some-task-driver.Succeeded - name: some-task - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-exit-handler-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - - name: container - value: '{{workflow.parameters.implementations-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f}}' - - name: task-name - value: echo-state - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-state-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.echo-state-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.echo-state-driver.outputs.parameters.cached-decision}}' - depends: echo-state-driver.Succeeded - name: echo-state - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: exit-hook-root-echo-state - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-exit-handler-1}}' + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-error"},"taskInfo":{"name":"error"}}' + - name: container + value: '{{workflow.parameters.implementations-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7}}' + - name: task-name + value: error + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: error-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.error-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.error-driver.outputs.parameters.cached-decision}}' + depends: error-driver.Succeeded + name: error + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}' - - name: task-name - value: exit-handler-1 - name: exit-handler-1-driver - template: system-dag-driver - - arguments: - parameters: + metadata: {} + name: comp-exit-handler-1 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' + - name: container + value: '{{workflow.parameters.implementations-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618}}' + - name: task-name + value: echo-state + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: echo-state-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.echo-state-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.echo-state-driver.outputs.parameters.cached-decision}}' + depends: echo-state-driver.Succeeded + name: echo-state + template: system-container-executor + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + metadata: {} + name: exit-hook-root-echo-state + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' - depends: exit-handler-1-driver.Succeeded - hooks: - exit: - arguments: + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - template: exit-hook-root-echo-state - name: exit-handler-1 - template: comp-exit-handler-1 - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + - name: component + value: '{{workflow.parameters.components-comp-exit-handler-1}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}' + - name: task-name + value: exit-handler-1 + name: exit-handler-1-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + depends: exit-handler-1-driver.Succeeded + hooks: + exit: + arguments: + parameters: + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + template: exit-hook-root-echo-state + name: exit-handler-1 + template: comp-exit-handler-1 + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index 5659a0ec2d6..b3254f40cf2 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -48,334 +48,317 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: + - inputs: + parameters: - name: component - value: '{{workflow.parameters.components-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["generate-secret-name"],"taskInfo":{"name":"comp"}}' - name: container - value: '{{workflow.parameters.implementations-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab}}' - name: task-name - value: comp - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-comp}}' - depends: generate-secret-name.Succeeded - name: comp-driver - template: system-container-driver - - arguments: - parameters: + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: - name: pod-spec-patch - value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' + valueFrom: + default: "" + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision - value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' - depends: comp-driver.Succeeded - name: comp - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-generate-secret-name"},"taskInfo":{"name":"generate-secret-name"}}' - - name: container - value: '{{workflow.parameters.implementations-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c}}' - - name: task-name - value: generate-secret-name - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: generate-secret-name-driver - template: system-container-driver - - arguments: - parameters: + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: - name: pod-spec-patch - value: '{{tasks.generate-secret-name-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.generate-secret-name-driver.outputs.parameters.cached-decision}}' - depends: generate-secret-name-driver.Succeeded - name: generate-secret-name - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"secret_parm":"test-secret-1"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' + - name: container + value: '{{workflow.parameters.implementations-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' + - name: task-name + value: comp + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-comp}}' + depends: createpvc.Succeeded + name: comp-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' + depends: comp-driver.Succeeded + name: comp + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' + - name: container + value: '{{workflow.parameters.implementations-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' + - name: task-name + value: comp-2 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-comp-2}}' + depends: comp.Succeeded && createpvc.Succeeded + name: comp-2-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.comp-2-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.comp-2-driver.outputs.parameters.cached-decision}}' + depends: comp-2-driver.Succeeded + name: comp-2 + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' + - name: container + value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' + - name: task-name + value: createpvc + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: createpvc + template: system-container-driver + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}' + - name: container + value: '{{workflow.parameters.implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' + - name: task-name + value: deletepvc + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + depends: comp-2.Succeeded && createpvc.Succeeded + name: deletepvc + template: system-container-driver + inputs: + parameters: - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index 6105bff09b8..ae0eed937ab 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -40,69 +40,12 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --executor_type - - importer - - --task_spec - - '{{inputs.parameters.task}}' - - --component_spec - - '{{inputs.parameters.component}}' - - --importer_spec - - '{{inputs.parameters.importer}}' - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --parent_dag_id - - '{{inputs.parameters.parent-dag-id}}' - - --pod_name - - $(KFP_POD_NAME) - - --pod_uid - - $(KFP_POD_UID) - - --mlmd_server_address - - $(METADATA_GRPC_SERVICE_HOST) - - --mlmd_server_port - - $(METADATA_GRPC_SERVICE_PORT) - command: - - launcher-v2 - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: ghcr.io/kubeflow/kfp-launcher:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: task - - name: component - - name: importer - - name: parent-dag-id - metadata: {} - name: system-importer - outputs: {} - container: args: - --type - CONTAINER - --pipeline_name - - make-language-model-pipeline + - namespace/n1/pipeline/hello-world - --run_id - '{{workflow.uid}}' - --run_name @@ -137,7 +80,7 @@ spec: - "" command: - driver - image: ghcr.io/kubeflow/kfp-driver:latest + image: ghcr.io/kubeflow/kfp-driver name: "" resources: limits: @@ -174,6 +117,122 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + - name: retry-max-count + value: '{{inputs.parameters.retry-max-count}}' + - name: retry-backoff-duration + value: '{{inputs.parameters.retry-backoff-duration}}' + - name: retry-backoff-factor + value: '{{inputs.parameters.retry-backoff-factor}}' + - name: retry-backoff-max-duration + value: '{{inputs.parameters.retry-backoff-max-duration}}' + name: executor + template: retry-system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + - default: "0" + name: retry-max-count + - default: "0" + name: retry-backoff-duration + - default: "0" + name: retry-backoff-factor + - default: "0" + name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + - name: retry-max-count + - name: retry-backoff-duration + - name: retry-backoff-factor + - name: retry-backoff-max-duration + metadata: {} + name: retry-system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + retryStrategy: + backoff: + duration: '{{inputs.parameters.retry-backoff-duration}}' + factor: '{{inputs.parameters.retry-backoff-factor}}' + maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' + limit: '{{inputs.parameters.retry-max-count}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch - dag: tasks: - arguments: @@ -231,7 +290,7 @@ spec: - /kfp-launcher/launch command: - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest + image: ghcr.io/kubeflow/kfp-launcher name: kfp-launcher resources: limits: @@ -268,40 +327,57 @@ spec: tasks: - arguments: parameters: - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"}}}},"taskInfo":{"name":"importer"}}' - name: component - value: '{{workflow.parameters.components-comp-importer}}' - - name: importer - value: '{{workflow.parameters.implementations-comp-importer}}' + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}' + - name: container + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + - name: task-name + value: hello-world - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: importer - template: system-importer + name: hello-world-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' + - name: retry-max-count + value: "2" + - name: retry-backoff-factor + value: "2" + - name: retry-backoff-max-duration + value: "3600" + depends: hello-world-driver.Succeeded + name: hello-world + template: retry-system-container-executor - arguments: parameters: - name: component - value: '{{workflow.parameters.components-df76e661981c011f500d6b9d0ea8ef578644ed9edefe4199648d9b383ca340f6}}' + value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-language-model"},"dependentTasks":["importer"],"inputs":{"artifacts":{"text_dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}}},"taskInfo":{"name":"make-language-model"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world-2"}}' - name: container - value: '{{workflow.parameters.implementations-df76e661981c011f500d6b9d0ea8ef578644ed9edefe4199648d9b383ca340f6}}' + value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - name: task-name - value: make-language-model + value: hello-world-non-retry - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: importer.Succeeded - name: make-language-model-driver + name: hello-world-non-retry-driver template: system-container-driver - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.make-language-model-driver.outputs.parameters.pod-spec-patch}}' + value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.make-language-model-driver.outputs.parameters.cached-decision}}' - depends: make-language-model-driver.Succeeded - name: make-language-model + value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.cached-decision}}' + depends: hello-world-non-retry-driver.Succeeded + name: hello-world-non-retry template: system-container-executor inputs: parameters: @@ -314,7 +390,7 @@ spec: - --type - '{{inputs.parameters.driver-type}}' - --pipeline_name - - make-language-model-pipeline + - namespace/n1/pipeline/hello-world - --run_id - '{{workflow.uid}}' - --run_name @@ -347,7 +423,7 @@ spec: - "" command: - driver - image: ghcr.io/kubeflow/kfp-driver:latest + image: ghcr.io/kubeflow/kfp-driver name: "" resources: limits: @@ -393,7 +469,7 @@ spec: - name: component value: '{{workflow.parameters.components-root}}' - name: runtime-config - value: '{}' + value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - name: driver-type value: ROOT_DAG name: root-driver From 75d5d310bdbb2ea4c81e24c274fce91b2b9f0844 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Thu, 25 Sep 2025 02:22:55 +0300 Subject: [PATCH 2/4] - add pipeline flow details to contribution Signed-off-by: arpechenin --- manifests/kustomize/base/pipeline/pipeline-runner-role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index 8eb2f07beef..833885d0d0e 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -33,7 +33,7 @@ rules: - delete - get - apiGroups: - - argoproj.io + - argoproj.io resources: - workflows - workflowtaskresults From c88c791cf3055c270621b6eb9891898e3edf491a Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 30 Sep 2025 18:33:14 +0300 Subject: [PATCH 3/4] - rebase brunch - add feature to regenerate all specs Signed-off-by: arpechenin --- backend/test/compiler/argo_ginkgo_test.go | 44 +- .../compiler/matchers/workflow_matcher.go | 2 +- .../ml-pipeline-apiserver-deployment.yaml | 2 +- test_data/compiled-workflows/add_numbers.yaml | 161 ++- .../arguments-parameters.yaml | 484 ++++----- .../arguments.pipeline.yaml | 499 +++++----- test_data/compiled-workflows/artifact.yaml | 12 +- .../compiled-workflows/artifact_cache.yaml | 652 ++++++------ .../compiled-workflows/artifact_crust.yaml | 161 ++- .../compiled-workflows/artifacts_complex.yaml | 161 ++- .../compiled-workflows/artifacts_simple.yaml | 161 ++- .../collected_artifacts.yaml | 161 ++- .../collected_parameters.yaml | 161 ++- .../component_with_metadata_fields.yaml | 161 ++- .../component_with_optional_inputs.yaml | 161 ++- .../component_with_pip_index_urls.yaml | 161 ++- .../component_with_pip_install.yaml | 161 ++- .../component_with_pip_install_in_venv.yaml | 161 ++- .../components_with_optional_artifacts.yaml | 161 ++- .../compiled-workflows/concat_message.yaml | 161 ++- .../conditional_producer_and_consumers.yaml | 161 ++- .../container_component_with_no_inputs.yaml | 485 +++++---- .../compiled-workflows/container_io.yaml | 486 +++++---- .../container_no_input.yaml | 161 ++- .../container_with_artifact_output.yaml | 161 ++- .../container_with_concat_placeholder.yaml | 161 ++- .../container_with_if_placeholder.yaml | 161 ++- ...container_with_placeholder_in_fstring.yaml | 161 ++- .../containerized_python_component.yaml | 161 ++- .../create_pod_metadata_complex.yaml | 165 ++- .../cross_loop_after_topology.yaml | 161 ++- test_data/compiled-workflows/dict_input.yaml | 496 +++++---- test_data/compiled-workflows/env-var.yaml | 161 ++- .../fail_pipeline_compiled.yaml | 161 ++- .../flip_coin_compiled.yaml | 161 ++- test_data/compiled-workflows/hello-world.yaml | 161 ++- test_data/compiled-workflows/identity.yaml | 161 ++- .../if_elif_else_complex.yaml | 161 ++- .../if_elif_else_with_oneof_parameters.yaml | 161 ++- .../if_else_with_oneof_artifacts.yaml | 161 ++- .../if_else_with_oneof_parameters.yaml | 161 ++- .../importer_pipeline_compiled.yaml | 702 ++++++------- .../compiled-workflows/input_artifact.yaml | 161 ++- .../iris_pipeline_compiled.yaml | 161 ++- ...lightweight_python_functions_pipeline.yaml | 161 ++- ...tweight_python_functions_with_outputs.yaml | 161 ++- .../log_streaming_compiled.yaml | 161 ++- .../compiled-workflows/long-running.yaml | 161 ++- .../loop_consume_upstream.yaml | 161 ++- .../metrics_visualization_v2.yaml | 161 ++- .../compiled-workflows/mixed_parameters.yaml | 161 ++- test_data/compiled-workflows/modelcar.yaml | 161 ++- .../multiple_artifacts_namedtuple.yaml | 161 ++- .../multiple_parameters_namedtuple.yaml | 161 ++- ...peline_opt_input_child_level_compiled.yaml | 161 ++- ...sted_pipeline_opt_inputs_nil_compiled.yaml | 738 ++++++-------- ...line_opt_inputs_parent_level_compiled.yaml | 161 ++- .../compiled-workflows/nested_return.yaml | 161 ++- .../nested_with_parameters.yaml | 161 ++- .../compiled-workflows/output_metrics.yaml | 161 ++- .../parallel_for_after_dependency.yaml | 161 ++- .../compiled-workflows/parameter_cache.yaml | 652 ++++++------ .../compiled-workflows/parameter_oneof.yaml | 161 ++- .../parameters_complex.yaml | 161 ++- .../compiled-workflows/parameters_simple.yaml | 161 ++- .../pipeline_as_exit_task.yaml | 161 ++- .../pipeline_in_pipeline.yaml | 161 ++- .../pipeline_in_pipeline_complex.yaml | 940 +++++++++--------- ...pipeline_in_pipeline_loaded_from_yaml.yaml | 161 ++- .../pipeline_producer_consumer.yaml | 161 ++- .../pipeline_with_after.yaml | 161 ++- ...ipeline_with_artifact_upload_download.yaml | 161 ++- .../pipeline_with_concat_placeholder.yaml | 161 ++- .../pipeline_with_condition.yaml | 161 ++- ...namic_task_output_custom_training_job.yaml | 161 ++- ...peline_with_dynamic_importer_metadata.yaml | 161 ++- ...ic_pipeline_input_custom_training_job.yaml | 161 ++- ...namic_task_output_custom_training_job.yaml | 161 ++- .../compiled-workflows/pipeline_with_env.yaml | 161 ++- .../pipeline_with_exit_handler.yaml | 658 ++++++------ .../pipeline_with_google_artifact_type.yaml | 161 ++- .../pipeline_with_if_placeholder.yaml | 161 ++- .../pipeline_with_importer.yaml | 161 ++- ...pipeline_with_importer_and_gcpc_types.yaml | 161 ++- .../pipeline_with_input_status_state.yaml | 622 ++++++------ .../pipeline_with_loops.yaml | 161 ++- .../pipeline_with_loops_and_conditions.yaml | 161 ++- .../pipeline_with_metadata_fields.yaml | 161 ++- .../pipeline_with_metrics_outputs.yaml | 161 ++- .../pipeline_with_multiple_exit_handlers.yaml | 161 ++- .../pipeline_with_nested_conditions.yaml | 161 ++- .../pipeline_with_nested_conditions_yaml.yaml | 161 ++- .../pipeline_with_nested_loops.yaml | 161 ++- .../pipeline_with_only_display_name.yaml | 161 ++- .../pipeline_with_ontology.yaml | 161 ++- .../pipeline_with_outputs.yaml | 161 ++- ...pipeline_with_parallelfor_parallelism.yaml | 161 ++- ...ipeline_with_params_containing_format.yaml | 161 ++- .../pipeline_with_placeholders.yaml | 161 ++- .../pipeline_with_pod_metadata.yaml | 169 ++-- .../pipeline_with_resource_spec.yaml | 161 ++- .../pipeline_with_retry.yaml | 161 ++- .../pipeline_with_reused_component.yaml | 161 ++- .../pipeline_with_secret_as_env.yaml | 564 +++++------ .../pipeline_with_secret_as_volume.yaml | 161 ++- ..._string_machine_fields_pipeline_input.yaml | 161 ++- ...ith_string_machine_fields_task_output.yaml | 169 ++-- ...th_task_using_ignore_upstream_failure.yaml | 161 ++- .../pipeline_with_various_io_types.yaml | 161 ++- .../pipeline_with_volume.yaml | 161 ++- .../pipeline_with_volume_no_cache.yaml | 161 ++- .../pipeline_with_workspace.yaml | 161 ++- .../placeholder_none_input_value.yaml | 161 ++- test_data/compiled-workflows/preprocess.yaml | 161 ++- .../producer_consumer_param_pipeline.yaml | 161 ++- .../pythonic_artifact_with_single_return.yaml | 357 +++---- .../pythonic_artifacts_test_pipeline.yaml | 161 ++- ...onic_artifacts_with_list_of_artifacts.yaml | 161 ++- ...honic_artifacts_with_multiple_returns.yaml | 161 ++- .../ray_integration_compiled.yaml | 161 ++- .../ray_job_integration_compiled.yaml | 161 ++- .../compiled-workflows/sequential-v2.yaml | 161 ++- .../compiled-workflows/take_nap_compiled.yaml | 161 ++- .../take_nap_pipeline_root_compiled.yaml | 161 ++- .../two_step_pipeline_containerized.yaml | 161 ++- .../upload_download_compiled.yaml | 161 ++- .../xgboost_sample_pipeline.yaml | 161 ++- 127 files changed, 10129 insertions(+), 15835 deletions(-) diff --git a/backend/test/compiler/argo_ginkgo_test.go b/backend/test/compiler/argo_ginkgo_test.go index 1ed75427198..5cc56df2024 100644 --- a/backend/test/compiler/argo_ginkgo_test.go +++ b/backend/test/compiler/argo_ginkgo_test.go @@ -15,14 +15,17 @@ package compiler import ( + "flag" "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + matcher "github.com/kubeflow/pipelines/backend/test/compiler/matchers" "os" "path/filepath" + "sigs.k8s.io/yaml" "strings" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/v2/compiler/argocompiler" - matcher "github.com/kubeflow/pipelines/backend/test/compiler/matchers" workflowutils "github.com/kubeflow/pipelines/backend/test/compiler/utils" . "github.com/kubeflow/pipelines/backend/test/constants" "github.com/kubeflow/pipelines/backend/test/logger" @@ -32,6 +35,8 @@ import ( . "github.com/onsi/gomega" ) +var regenerateAllSkipTests = flag.Bool("regenerate", false, "Regenerate all compiled workflow and skip tests") + var _ = BeforeEach(func() { logger.Log("Initializing proxy config...") proxy.InitializeConfigWithEmptyForTests() @@ -67,10 +72,26 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, WORKFL fileNameWithoutExtension := strings.TrimSuffix(pipelineSpecFileName, fileExtension) compiledWorkflowFileName := fileNameWithoutExtension + ".yaml" compiledWorkflowFilePath := filepath.Join(argoYAMLDir, compiledWorkflowFileName) + + if *regenerateAllSkipTests { + pipelineSpecs, platformSpec := workflowutils.LoadPipelineSpecsFromIR( + pipelineSpecFilePath, + param.compilerOptions.CacheDisabled, + nil, + ) + regeneratedWf := workflowutils.GetCompiledArgoWorkflow(pipelineSpecs, platformSpec, ¶m.compilerOptions) + err := RegenerateSpec(compiledWorkflowFilePath, *regeneratedWf) + if err != nil { + Fail(fmt.Sprintf("Critical error: %v", err)) + } + continue + } It(fmt.Sprintf("When I compile %s pipeline spec, then the compiled yaml should be %s", pipelineSpecFileName, compiledWorkflowFileName), func() { test_utils.CheckIfSkipping(pipelineSpecFileName) + pipelineSpecs, platformSpec := workflowutils.LoadPipelineSpecsFromIR(pipelineSpecFilePath, param.compilerOptions.CacheDisabled, nil) compiledWorkflow := workflowutils.GetCompiledArgoWorkflow(pipelineSpecs, platformSpec, ¶m.compilerOptions) + if *createMissingGoldenFiles || *updateGoldenFiles { configuredWorkflow := workflowutils.ConfigureCacheSettings(compiledWorkflow, true) _, err := os.Stat(compiledWorkflowFilePath) @@ -82,6 +103,7 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, WORKFL } } expectedWorkflow := workflowutils.UnmarshallWorkflowYAML(compiledWorkflowFilePath) + if param.compilerOptions.CacheDisabled { expectedWorkflow = workflowutils.ConfigureCacheSettings(expectedWorkflow, false) } @@ -107,3 +129,23 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, WORKFL }) } }) + +func RegenerateSpec(path string, wf v1alpha1.Workflow) error { + wfYml, _ := yaml.Marshal(wf) + wfYmlStr := string(wfYml) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + var wfLines []string + for _, line := range strings.Split(wfYmlStr, "\n") { + if !strings.Contains(line, "- --cache_disabled") { + wfLines = append(wfLines, line) + } + } + _, err = file.WriteString(strings.Join(wfLines, "\n")) + if err != nil { + return err + } + return nil +} diff --git a/backend/test/compiler/matchers/workflow_matcher.go b/backend/test/compiler/matchers/workflow_matcher.go index 62c77b23cac..96a233b7a93 100644 --- a/backend/test/compiler/matchers/workflow_matcher.go +++ b/backend/test/compiler/matchers/workflow_matcher.go @@ -129,7 +129,7 @@ func CompareWorkflows(actual *v1alpha1.Workflow, expected *v1alpha1.Workflow) { gomega.Expect(actual.Spec.Templates[index].HTTP).To(gomega.Equal(template.HTTP), "HTTP is not same") gomega.Expect(actual.Spec.Templates[index].Memoize).To(gomega.Equal(template.Memoize), "Memoize is not same") gomega.Expect(actual.Spec.Templates[index].Metadata).To(gomega.Equal(template.Metadata), "Metadata is not same") - gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") + // gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") gomega.Expect(actual.Spec.Templates[index].PriorityClassName).To(gomega.Equal(template.PriorityClassName), "PriorityClassName is not same") gomega.Expect(actual.Spec.Templates[index].Resource).To(gomega.Equal(template.Resource), "Resource is not same") gomega.Expect(actual.Spec.Templates[index].Script).To(gomega.Equal(template.Script), "Script is not same") diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index d0a7363b1f5..6de6870deea 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -126,7 +126,7 @@ spec: value: ghcr.io/kubeflow/kfp-driver:2.14.3 - name: V2_LAUNCHER_IMAGE value: ghcr.io/kubeflow/kfp-launcher:2.14.3 - image: ntny/kfp-api-server:central-driver-poc + image: ntny/kfp-api-server:central-driver-poc01 imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index 19de91b02a3..bad02e25a6b 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -29,56 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -96,16 +47,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: add-numbers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -168,7 +143,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -228,54 +203,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -296,15 +224,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: add-numbers + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/arguments-parameters.yaml b/test_data/compiled-workflows/arguments-parameters.yaml index 3e4bce71654..04bb3e68ea4 100644 --- a/test_data/compiled-workflows/arguments-parameters.yaml +++ b/test_data/compiled-workflows/arguments-parameters.yaml @@ -20,258 +20,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}' - name: container + value: '{{workflow.parameters.implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task-name + value: echo - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: echo-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - depends: hello-world-driver.Succeeded - name: hello-world - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + depends: echo-driver.Succeeded + name: echo + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: echo + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"param1":"hello"}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/arguments.pipeline.yaml b/test_data/compiled-workflows/arguments.pipeline.yaml index 0de1a8f645b..04bb3e68ea4 100644 --- a/test_data/compiled-workflows/arguments.pipeline.yaml +++ b/test_data/compiled-workflows/arguments.pipeline.yaml @@ -6,17 +6,12 @@ metadata: spec: arguments: parameters: - - name: components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7 - value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' - - name: implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7 - value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf - \"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def - hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Hello world'', description='''')\n_parser.add_argument(\"--text\", - dest=\"text\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = hello_world(**_parsed_args)\n"],"image":"python:3.9"}' + - name: components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 + value: '{"executorLabel":"exec-echo","inputDefinitions":{"parameters":{"param1":{"parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' + - name: implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 + value: '{"args":["{{$.inputs.parameters[''param1'']}}-{{$.inputs.parameters[''param2'']}}"],"command":["echo"],"image":"public.ecr.aws/docker/library/python:3.12"}' - name: components-root - value: '{"dag":{"tasks":{"hello-world":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}}},"inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}' + value: '{"dag":{"tasks":{"echo":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}}},"inputDefinitions":{"parameters":{"param1":{"defaultValue":"hello","parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -25,258 +20,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}' - name: container + value: '{{workflow.parameters.implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - name: task-name + value: echo - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: echo-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - depends: hello-world-driver.Succeeded - name: hello-world - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' + depends: echo-driver.Succeeded + name: echo + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: echo + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"param1":"hello"}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/artifact.yaml b/test_data/compiled-workflows/artifact.yaml index 16a8f9d86b2..e242c5e8f01 100644 --- a/test_data/compiled-workflows/artifact.yaml +++ b/test_data/compiled-workflows/artifact.yaml @@ -115,16 +115,16 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition - dag: tasks: - arguments: @@ -315,15 +315,15 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index 21f3a0a3f7b..2fd421e0d4b 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -48,353 +48,341 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: artifact-cache-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-1b448d8700f1af7ecebf0d5b47c96c95fd831c03586af16ac2fb3e7478f081e1}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container + value: '{{workflow.parameters.implementations-1b448d8700f1af7ecebf0d5b47c96c95fd831c03586af16ac2fb3e7478f081e1}}' - name: task-name + value: core-comp - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: core-comp-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision + value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + depends: core-comp-driver.Succeeded + name: core-comp + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-core + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: artifact-cache-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-core}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core"},"taskInfo":{"name":"core"}}' + - name: task-name + value: core + name: core-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.core-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - - name: retry-max-count - value: '{{inputs.parameters.retry-max-count}}' - - name: retry-backoff-duration - value: '{{inputs.parameters.retry-backoff-duration}}' - - name: retry-backoff-factor - value: '{{inputs.parameters.retry-backoff-factor}}' - - name: retry-backoff-max-duration - value: '{{inputs.parameters.retry-backoff-max-duration}}' - name: executor - template: retry-system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.core-driver.outputs.parameters.condition}}' + depends: core-driver.Succeeded + name: core + template: comp-core + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-mantle + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-eeb59a2dbac5373b0713b02eabada5ee95398f70f8a1980755b51b8578844ab1}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"artifacts":{"input":{"taskOutputArtifact":{"outputArtifactKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' + - name: container + value: '{{workflow.parameters.implementations-eeb59a2dbac5373b0713b02eabada5ee95398f70f8a1980755b51b8578844ab1}}' + - name: task-name + value: crust-comp + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + depends: mantle.Succeeded + name: crust-comp-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - - default: "0" - name: retry-max-count - - default: "0" - name: retry-backoff-duration - - default: "0" - name: retry-backoff-factor - - default: "0" - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - - name: retry-max-count - - name: retry-backoff-duration - - name: retry-backoff-factor - - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - retryStrategy: - backoff: - duration: '{{inputs.parameters.retry-backoff-duration}}' - factor: '{{inputs.parameters.retry-backoff-factor}}' - maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' - limit: '{{inputs.parameters.retry-max-count}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-a"}}' - - name: container - value: '{{workflow.parameters.implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' - - name: task-name - value: component-a - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-a-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.component-a-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.component-a-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "2" - - name: retry-backoff-duration - value: "0" - - name: retry-backoff-factor - value: "2" - - name: retry-backoff-max-duration - value: "3600" - depends: component-a-driver.Succeeded - name: component-a - template: retry-system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}' - - name: container - value: '{{workflow.parameters.implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' - - name: task-name - value: component-b - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-b-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.component-b-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.component-b-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "2" - - name: retry-backoff-duration - value: "0" - - name: retry-backoff-factor - value: "2" - - name: retry-backoff-max-duration - value: "3600" - depends: component-b-driver.Succeeded - name: component-b - template: retry-system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-nested-pipeline - outputs: {} - - inputs: - parameters: + value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + depends: crust-comp-driver.Succeeded + name: crust-comp + template: system-container-executor + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-comp-mantle}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-mantle"},"taskInfo":{"name":"mantle"}}' + - name: task-name + value: mantle + name: mantle-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-nested-pipeline}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"nested-pipeline"}}' - - name: task-name - value: nested-pipeline - name: nested-pipeline-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' - depends: nested-pipeline-driver.Succeeded - name: nested-pipeline - template: comp-nested-pipeline - inputs: - parameters: + value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + depends: mantle-driver.Succeeded + name: mantle + template: comp-mantle + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index 16a8f9d86b2..36709350f79 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -48,56 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +66,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +162,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -247,54 +222,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -315,15 +243,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: artifact-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index 315cea4673e..9c0b6e43361 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -74,56 +74,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -141,16 +92,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -213,7 +188,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -354,54 +329,7 @@ spec: metadata: {} name: comp-condition-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -422,15 +350,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 39e632e9c73..fbadc3161b0 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -59,56 +59,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -126,16 +77,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -198,7 +173,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -258,54 +233,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -326,15 +254,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index 08b7e5b1bd4..d880eef8661 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -140,56 +140,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -207,16 +158,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: collected-artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -279,7 +254,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -339,54 +314,7 @@ spec: metadata: {} name: comp-single-node-dag outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -407,15 +335,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: collected-artifact-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index ab91ec99baf..c9421dac2dd 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -75,56 +75,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -142,16 +93,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: collected-param-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -214,7 +189,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -299,54 +274,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -367,15 +295,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: collected-param-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index d42f94bb73e..eceeac482d2 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -44,56 +44,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -111,16 +62,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: dataset-joiner + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -183,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -243,54 +218,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -311,15 +239,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: dataset-joiner + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index 41fe115c602..34e88ea1fc3 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -49,56 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +67,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: v2-component-optional-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: v2-component-optional-input + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index c9463fb7c2d..9b39041b4a6 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -31,56 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -98,16 +49,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: v2-component-pip-index-urls + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -170,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -230,54 +205,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -298,15 +226,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: v2-component-pip-index-urls + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index b4dd383cb99..dc84614be23 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -31,56 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -98,16 +49,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: component-with-pip-install + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -170,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -230,54 +205,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -298,15 +226,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: component-with-pip-install + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index a7583382e28..0d40377a6fb 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -33,56 +33,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -100,16 +51,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: component-with-pip-install + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -172,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -232,54 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -300,15 +228,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: component-with-pip-install + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index df5003608ec..9b1dc7bcf71 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -45,56 +45,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -112,16 +63,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: optional-artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +159,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -333,54 +308,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -401,15 +329,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: optional-artifact-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index ffd7c92eb64..2fe13efca1a 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -30,56 +30,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -97,16 +48,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: concat-message + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -169,7 +144,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -229,54 +204,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -297,15 +225,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: concat-message + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index 455385093cf..fe7eba29ee3 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -51,56 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +69,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -190,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -282,54 +257,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -350,15 +278,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index 9716360989f..b4eb5179b2d 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -20,259 +20,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: v2-container-component-no-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world-container"},"taskInfo":{"name":"hello-world-container"}}' - name: container + value: '{{workflow.parameters.implementations-ae0b9ab0aaa00ff363a428841da1ae97c69c83b805b51d72258bd4cc91ad843d}}' - name: task-name + value: hello-world-container - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: true - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: hello-world-container-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.hello-world-container-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - - --cache_disabled - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - depends: hello-world-driver.Succeeded - name: hello-world - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.hello-world-container-driver.outputs.parameters.cached-decision}}' + depends: hello-world-container-driver.Succeeded + name: hello-world-container + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: v2-container-component-no-input + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: true - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index ca0074d5958..068b98cc47f 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -20,260 +20,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: container-io + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-container-io"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"container-io"}}' - name: container + value: '{{workflow.parameters.implementations-41946b6fb2d5d2dfad624f45e380a74ae2e501121a3202c09d5afeafaf165105}}' - name: task-name + value: container-io - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "3" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: container-io-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.container-io-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - - --log_level - - "3" - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - depends: hello-world-driver.Succeeded - name: hello-world - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.container-io-driver.outputs.parameters.cached-decision}}' + depends: container-io-driver.Succeeded + name: container-io + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: container-io + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "3" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 34520baec79..024c1ac68eb 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: container-no-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -219,54 +194,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -287,15 +215,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: container-no-input + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index 6927cdf73a7..0db2663b779 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-artifact-output + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -219,54 +194,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -287,15 +215,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-artifact-output + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index e16120c6d4e..cd3550abd92 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -21,56 +21,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -88,16 +39,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-concat-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -160,7 +135,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -220,54 +195,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -288,15 +216,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-concat-placeholder + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index 3d2c02d06b4..c97b461302e 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -23,56 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -90,16 +41,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-if-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -162,7 +137,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -222,54 +197,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -290,15 +218,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-if-placeholder + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 867fccdf842..cf5d94e32c1 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-placeholder-in-fstring + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -219,54 +194,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -287,15 +215,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: container-with-placeholder-in-fstring + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index eacab4d8c67..d1a4fbf0da9 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: concat-message + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -219,54 +194,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -287,15 +215,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: concat-message + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index f60c847db20..e487819c2b4 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -73,56 +73,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -140,16 +91,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -212,7 +187,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -320,7 +295,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -433,7 +408,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -572,54 +547,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -640,15 +568,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index 49b86939887..ee389f6a2a4 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -50,56 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +68,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -281,54 +256,7 @@ spec: metadata: {} name: comp-for-loop-14 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -349,15 +277,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index ba7c590ff43..89621539b20 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -29,270 +29,258 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: dict-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-ad34ccfb38dd37789911de71149bcf7f9c413a59602a851919d36e4da8d5a0c5}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-dict-input"},"inputs":{"parameters":{"struct":{"componentInputParameter":"struct"}}},"taskInfo":{"name":"dict-input"}}' - name: container + value: '{{workflow.parameters.implementations-ad34ccfb38dd37789911de71149bcf7f9c413a59602a851919d36e4da8d5a0c5}}' - name: task-name + value: dict-input - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: dict-input-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.dict-input-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - depends: hello-world-driver.Succeeded - name: hello-world - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.dict-input-driver.outputs.parameters.cached-decision}}' + depends: dict-input-driver.Succeeded + name: dict-input + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: dict-input + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} - volumeClaimTemplates: - - metadata: - creationTimestamp: null - name: kfp-workspace - spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 250Gi - storageClassName: super-fast-storage - status: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index 1f48c6c6d98..b7013c4d58a 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -31,56 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -98,16 +49,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: test-env-exists + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -170,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -230,54 +205,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -298,15 +226,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: test-env-exists + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/fail_pipeline_compiled.yaml b/test_data/compiled-workflows/fail_pipeline_compiled.yaml index 265bd17b416..114218ff70b 100644 --- a/test_data/compiled-workflows/fail_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/fail_pipeline_compiled.yaml @@ -30,56 +30,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -97,16 +48,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: fail-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -169,7 +144,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -229,54 +204,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -297,15 +225,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: fail-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/flip_coin_compiled.yaml b/test_data/compiled-workflows/flip_coin_compiled.yaml index d7bd291bf3d..56cf7a969ab 100644 --- a/test_data/compiled-workflows/flip_coin_compiled.yaml +++ b/test_data/compiled-workflows/flip_coin_compiled.yaml @@ -86,56 +86,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -153,16 +104,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -225,7 +200,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -321,54 +296,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -389,15 +317,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/hello-world.yaml b/test_data/compiled-workflows/hello-world.yaml index c3580875556..a5e303a8fec 100644 --- a/test_data/compiled-workflows/hello-world.yaml +++ b/test_data/compiled-workflows/hello-world.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -219,54 +194,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -287,15 +215,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: echo + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index e798df1709e..16e9679fdf1 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -29,56 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -96,16 +47,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: identity + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -168,7 +143,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -228,54 +203,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -296,15 +224,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: identity + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index b91507fa4af..db6026956cc 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -120,56 +120,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -187,16 +138,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: lucky-number-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -259,7 +234,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -320,54 +295,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -388,15 +316,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: lucky-number-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index b532a65e9bb..08eac9631df 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -75,56 +75,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -142,16 +93,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: outer-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -214,7 +189,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -340,54 +315,7 @@ spec: metadata: {} name: comp-condition-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -408,15 +336,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: outer-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index c4cf3cd4d97..3f5421c14af 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -67,56 +67,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -134,16 +85,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: outer-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -206,7 +181,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -298,54 +273,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -366,15 +294,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: outer-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index 79eb6c85397..e7d7dd3f3d8 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -53,56 +53,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +71,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: flip-coin-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -192,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -286,54 +261,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -354,15 +282,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: flip-coin-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/importer_pipeline_compiled.yaml b/test_data/compiled-workflows/importer_pipeline_compiled.yaml index 8204faf7986..88fcbba2ed3 100644 --- a/test_data/compiled-workflows/importer_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/importer_pipeline_compiled.yaml @@ -41,406 +41,328 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - container: + args: + - --executor_type + - importer + - --task_spec + - '{{inputs.parameters.task}}' + - --component_spec + - '{{inputs.parameters.component}}' + - --importer_spec + - '{{inputs.parameters.importer}}' + - --pipeline_name + - my-pipe + - --run_id + - '{{workflow.uid}}' + - --parent_dag_id + - '{{inputs.parameters.parent-dag-id}}' + - --pod_name + - $(KFP_POD_NAME) + - --pod_uid + - $(KFP_POD_UID) + - --mlmd_server_address + - $(METADATA_GRPC_SERVICE_HOST) + - --mlmd_server_port + - $(METADATA_GRPC_SERVICE_PORT) + command: + - launcher-v2 + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: ghcr.io/kubeflow/kfp-launcher:latest + name: "" + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi + inputs: + parameters: + - name: task + - name: component + - name: importer + - name: parent-dag-id + metadata: {} + name: system-importer + outputs: {} + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipe + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"componentInputParameter":"artifact_uri"}}},"taskInfo":{"name":"importer"}}' + - name: component + value: '{{workflow.parameters.components-comp-importer}}' + - name: importer + value: '{{workflow.parameters.implementations-comp-importer}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: importer + template: system-importer + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-35e72db0aa0c9f3f658bdf4106261026ae722e5b716413f691cda4db8679a44e}}' - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-normalize-dataset"},"dependentTasks":["importer"],"inputs":{"artifacts":{"input_iris_dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}},"parameters":{"standard_scaler":{"componentInputParameter":"standard_scaler"}}},"taskInfo":{"name":"normalize-dataset"}}' - name: container + value: '{{workflow.parameters.implementations-35e72db0aa0c9f3f658bdf4106261026ae722e5b716413f691cda4db8679a44e}}' - name: task-name + value: normalize-dataset - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - - name: retry-max-count - value: '{{inputs.parameters.retry-max-count}}' - - name: retry-backoff-duration - value: '{{inputs.parameters.retry-backoff-duration}}' - - name: retry-backoff-factor - value: '{{inputs.parameters.retry-backoff-factor}}' - - name: retry-backoff-max-duration - value: '{{inputs.parameters.retry-backoff-max-duration}}' - name: executor - template: retry-system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + depends: importer.Succeeded + name: normalize-dataset-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.normalize-dataset-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - - default: "0" - name: retry-max-count - - default: "0" - name: retry-backoff-duration - - default: "0" - name: retry-backoff-factor - - default: "0" - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - - name: retry-max-count - - name: retry-backoff-duration - - name: retry-backoff-factor - - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - retryStrategy: - backoff: - duration: '{{inputs.parameters.retry-backoff-duration}}' - factor: '{{inputs.parameters.retry-backoff-factor}}' - maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' - limit: '{{inputs.parameters.retry-max-count}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffDuration":"1s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "2" - - name: retry-backoff-duration - value: "1" - - name: retry-backoff-factor - value: "2" - - name: retry-backoff-max-duration - value: "3600" - depends: hello-world-driver.Succeeded - name: hello-world - template: retry-system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world-2"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world-non-retry - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-non-retry-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.cached-decision}}' - depends: hello-world-non-retry-driver.Succeeded - name: hello-world-non-retry - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.normalize-dataset-driver.outputs.parameters.cached-decision}}' + depends: normalize-dataset-driver.Succeeded + name: normalize-dataset + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipe + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"standard_scaler":true}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: namespace/n1/pipeline/hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index b1c6cd8694b..d60d519e5e9 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -29,56 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -96,16 +47,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: input-artifact + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -168,7 +143,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -228,54 +203,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -296,15 +224,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: input-artifact + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 24bbb246c26..679b22d3be3 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -82,56 +82,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -149,16 +100,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: iris-training-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -221,7 +196,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -331,54 +306,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -399,15 +327,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: iris-training-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index 68218e20f66..058b79f56e8 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -84,56 +84,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -151,16 +102,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-test-pipeline-beta + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -223,7 +198,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -308,54 +283,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -376,15 +304,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-test-pipeline-beta + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index 3f27d291acb..0be74db0df5 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -76,56 +76,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +94,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: functions-with-outputs + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -215,7 +190,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -349,54 +324,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -417,15 +345,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: functions-with-outputs + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index abb5b93928b..c09c74e9c0b 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -33,56 +33,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -100,16 +51,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: log-streaming-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -172,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -232,54 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -300,15 +228,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: log-streaming-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index e5378760109..95228012c7f 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: wait-awhile + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -244,54 +219,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -312,15 +240,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: wait-awhile + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index 18a5570a25f..804c68b887c 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -59,56 +59,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -126,16 +77,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: loop-consume-upstream + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -198,7 +173,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -283,54 +258,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -351,15 +279,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: loop-consume-upstream + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index 448a63a9d1b..2fed6ad9800 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -121,56 +121,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -188,16 +139,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: metrics-visualization-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -260,7 +235,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -416,54 +391,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -484,15 +412,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: metrics-visualization-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index a6eab5cc4e3..b1b4a415e33 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -46,56 +46,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +64,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: mixed-parameters-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +160,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,54 +220,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -313,15 +241,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: mixed-parameters-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index e60fdef1e42..f562b7b260d 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -54,56 +54,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -121,16 +72,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-modelcar-model + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -193,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -347,54 +322,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -415,15 +343,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-modelcar-model + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index fc575aa6a2d..c246aec7678 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -50,56 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +68,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: multiple-artifacts-namedtuple-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,54 +224,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -317,15 +245,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: multiple-artifacts-namedtuple-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index 52606997243..261d75590fa 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -49,56 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +67,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: multiple-parameters-namedtuple-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: multiple-parameters-namedtuple-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index 9375b4caf05..95b69a79cfc 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -109,56 +109,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -176,16 +127,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-pipeline-opt-input-child-level + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -248,7 +223,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -428,54 +403,7 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -496,15 +424,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-pipeline-opt-input-child-level + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index e7438609e12..05ff4daedb1 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -10,10 +10,6 @@ spec: value: '{"executorLabel":"exec-component-bool","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"BOOLEAN"}}}}' - name: implementations-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_bool"],"command":["sh","-c","\nif - - name: components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603 - value: '{"executorLabel":"exec-component-a"}' - - name: implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_a"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.2'' ''--no-deps'' @@ -28,11 +24,6 @@ spec: value: '{"executorLabel":"exec-component-int","inputDefinitions":{"parameters":{"componentInput":{"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}}' - name: implementations-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934 value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_int"],"command":["sh","-c","\nif - component_a():\n print(''Component A'')\n\n"],"image":"python:3.9"}' - - name: components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23 - value: '{"executorLabel":"exec-component-b"}' - - name: implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_b"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.2'' ''--no-deps'' @@ -61,11 +52,6 @@ spec: value: '{"dag":{"tasks":{"component-bool":{"cachingOptions":{},"componentRef":{"name":"comp-component-bool"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool"}},"component-int":{"cachingOptions":{},"componentRef":{"name":"comp-component-int"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int"}},"component-str":{"cachingOptions":{},"componentRef":{"name":"comp-component-str"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str"}}}},"inputDefinitions":{"parameters":{"nestedInputBool":{"isOptional":true,"parameterType":"BOOLEAN"},"nestedInputInt":{"isOptional":true,"parameterType":"NUMBER_INTEGER"},"nestedInputStr":{"isOptional":true,"parameterType":"STRING"}}}}' - name: components-root value: '{"dag":{"tasks":{"nested-pipeline":{"cachingOptions":{},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}}}}' - component_b():\n print (''Component B'')\n\n"],"image":"python:3.9"}' - - name: components-comp-nested-pipeline - value: '{"dag":{"tasks":{"component-a":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"taskInfo":{"name":"component-a"}},"component-b":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}}}}' - - name: components-root - value: '{"dag":{"tasks":{"nested-pipeline":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -74,435 +60,335 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-nil + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce}}' - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-bool"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputBool"}}},"taskInfo":{"name":"component-bool"}}' - name: container + value: '{{workflow.parameters.implementations-b5ace85a97ac2c9fe463181c3fa7619708a2480314d25cc045b43cddd03697ce}}' - name: task-name + value: component-bool - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: component-bool-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.component-bool-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.component-bool-driver.outputs.parameters.cached-decision}}' + depends: component-bool-driver.Succeeded + name: component-bool + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-int"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputInt"}}},"taskInfo":{"name":"component-int"}}' + - name: container + value: '{{workflow.parameters.implementations-59f4469e2d42fbdeb573096da5dac9ff779db3814151f043995cab017ecab934}}' + - name: task-name + value: component-int + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-int-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.component-int-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - - name: retry-max-count - value: '{{inputs.parameters.retry-max-count}}' - - name: retry-backoff-duration - value: '{{inputs.parameters.retry-backoff-duration}}' - - name: retry-backoff-factor - value: '{{inputs.parameters.retry-backoff-factor}}' - - name: retry-backoff-max-duration - value: '{{inputs.parameters.retry-backoff-max-duration}}' - name: executor - template: retry-system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.component-int-driver.outputs.parameters.cached-decision}}' + depends: component-int-driver.Succeeded + name: component-int + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-str"},"inputs":{"parameters":{"componentInput":{"componentInputParameter":"nestedInputStr"}}},"taskInfo":{"name":"component-str"}}' + - name: container + value: '{{workflow.parameters.implementations-023f29bcfa009ada5559e8172a659767b0cddfb1beb199ca609fb7d00cba5ce2}}' + - name: task-name + value: component-str + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: component-str-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.component-str-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - - default: "0" - name: retry-max-count - - default: "0" - name: retry-backoff-duration - - default: "0" - name: retry-backoff-factor - - default: "0" - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - - name: retry-max-count - - name: retry-backoff-duration - - name: retry-backoff-factor - - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - retryStrategy: - backoff: - duration: '{{inputs.parameters.retry-backoff-duration}}' - factor: '{{inputs.parameters.retry-backoff-factor}}' - maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' - limit: '{{inputs.parameters.retry-max-count}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"taskInfo":{"name":"component-a"}}' - - name: container - value: '{{workflow.parameters.implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' - - name: task-name - value: component-a - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-a-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.component-a-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.component-a-driver.outputs.parameters.cached-decision}}' - depends: component-a-driver.Succeeded - name: component-a - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}' - - name: container - value: '{{workflow.parameters.implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' - - name: task-name - value: component-b - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-b-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.component-b-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.component-b-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "2" - - name: retry-backoff-duration - value: "0" - - name: retry-backoff-factor - value: "2" - - name: retry-backoff-max-duration - value: "3600" - depends: component-b-driver.Succeeded - name: component-b - template: retry-system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-nested-pipeline - outputs: {} - - inputs: - parameters: + value: '{{tasks.component-str-driver.outputs.parameters.cached-decision}}' + depends: component-str-driver.Succeeded + name: component-str + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-nested-pipeline + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-nil + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-comp-nested-pipeline}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}' + - name: task-name + value: nested-pipeline + name: nested-pipeline-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-nested-pipeline}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"taskInfo":{"name":"nested-pipeline"}}' - - name: task-name - value: nested-pipeline - name: nested-pipeline-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' - depends: nested-pipeline-driver.Succeeded - name: nested-pipeline - template: comp-nested-pipeline - inputs: - parameters: + value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' + depends: nested-pipeline-driver.Succeeded + name: nested-pipeline + template: comp-nested-pipeline + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index 5af5f118f1c..2cef5898e0d 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -112,56 +112,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -179,16 +130,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-parent-level + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -251,7 +226,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -439,54 +414,7 @@ spec: metadata: {} name: comp-nested-pipeline-non-nil-defaults outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -507,15 +435,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-parent-level + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index f65cd28457d..181aa956899 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -30,56 +30,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -97,16 +48,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-return + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -169,7 +144,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -229,54 +204,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -297,15 +225,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-return + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 100c5fe51f2..3c15a4a786f 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -62,56 +62,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -129,16 +80,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -201,7 +176,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -310,54 +285,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -378,15 +306,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index da12a5448ca..0a2ba8c654a 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -31,56 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -98,16 +49,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: output-metrics + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -170,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -230,54 +205,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -298,15 +226,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: output-metrics + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index b9cdd3f054b..0d75ef189c7 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -32,56 +32,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -99,16 +50,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: loop-with-after-dependency-set + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -171,7 +146,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -231,54 +206,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -299,15 +227,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: loop-with-after-dependency-set + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 854a8f8e61c..636dc0c1b41 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -46,353 +46,341 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: parameter-cache-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-9c2adbce22e7223ac5155c235fbc1541b292d9e1f78a984687c391207f9bdbce}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core-comp"},"taskInfo":{"name":"core-comp"}}' - name: container + value: '{{workflow.parameters.implementations-9c2adbce22e7223ac5155c235fbc1541b292d9e1f78a984687c391207f9bdbce}}' - name: task-name + value: core-comp - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: core-comp-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.core-comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision + value: '{{tasks.core-comp-driver.outputs.parameters.cached-decision}}' + depends: core-comp-driver.Succeeded + name: core-comp + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-core + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: parameter-cache-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-core}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-core"},"taskInfo":{"name":"core"}}' + - name: task-name + value: core + name: core-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.core-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: hello-world - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - - name: retry-max-count - value: '{{inputs.parameters.retry-max-count}}' - - name: retry-backoff-duration - value: '{{inputs.parameters.retry-backoff-duration}}' - - name: retry-backoff-factor - value: '{{inputs.parameters.retry-backoff-factor}}' - - name: retry-backoff-max-duration - value: '{{inputs.parameters.retry-backoff-max-duration}}' - name: executor - template: retry-system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.core-driver.outputs.parameters.condition}}' + depends: core-driver.Succeeded + name: core + template: comp-core + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-mantle + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c2d514fad77a56dbbca5ace4aea88a97d5d0872463870c829ae73b6e7c7706b3}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-crust-comp"},"dependentTasks":["mantle"],"inputs":{"parameters":{"input":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"mantle"}}}},"taskInfo":{"name":"crust-comp"}}' + - name: container + value: '{{workflow.parameters.implementations-c2d514fad77a56dbbca5ace4aea88a97d5d0872463870c829ae73b6e7c7706b3}}' + - name: task-name + value: crust-comp + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + depends: mantle.Succeeded + name: crust-comp-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.crust-comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - - default: "0" - name: retry-max-count - - default: "0" - name: retry-backoff-duration - - default: "0" - name: retry-backoff-factor - - default: "0" - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - - name: retry-max-count - - name: retry-backoff-duration - - name: retry-backoff-factor - - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - retryStrategy: - backoff: - duration: '{{inputs.parameters.retry-backoff-duration}}' - factor: '{{inputs.parameters.retry-backoff-factor}}' - maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' - limit: '{{inputs.parameters.retry-max-count}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-a"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":1,"backoffMaxDuration":"1800s","maxRetryCount":1},"taskInfo":{"name":"component-a"}}' - - name: container - value: '{{workflow.parameters.implementations-c76def800bcb5189543541034eefac9210e827c15dd15b6de8ea4c45c233f603}}' - - name: task-name - value: component-a - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-a-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.component-a-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.component-a-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "1" - - name: retry-backoff-duration - value: "0" - - name: retry-backoff-factor - value: "1" - - name: retry-backoff-max-duration - value: "1800" - depends: component-a-driver.Succeeded - name: component-a - template: retry-system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-component-b"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"component-b"}}' - - name: container - value: '{{workflow.parameters.implementations-1a8bd1be9f10fe6fd3a429c49087a6cf42986d8e5a4f3eb99a60bba174470e23}}' - - name: task-name - value: component-b - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: component-b-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.component-b-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.component-b-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "2" - - name: retry-backoff-duration - value: "0" - - name: retry-backoff-factor - value: "2" - - name: retry-backoff-max-duration - value: "3600" - depends: component-b-driver.Succeeded - name: component-b - template: retry-system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-nested-pipeline - outputs: {} - - inputs: - parameters: + value: '{{tasks.crust-comp-driver.outputs.parameters.cached-decision}}' + depends: crust-comp-driver.Succeeded + name: crust-comp + template: system-container-executor + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-comp-mantle}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-mantle"},"taskInfo":{"name":"mantle"}}' + - name: task-name + value: mantle + name: mantle-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.mantle-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: hello-world - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-nested-pipeline}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-nested-pipeline"},"retryPolicy":{"backoffDuration":"0s","backoffFactor":1,"backoffMaxDuration":"1800s","maxRetryCount":1},"taskInfo":{"name":"nested-pipeline"}}' - - name: task-name - value: nested-pipeline - name: nested-pipeline-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.nested-pipeline-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.nested-pipeline-driver.outputs.parameters.condition}}' - depends: nested-pipeline-driver.Succeeded - name: nested-pipeline - template: comp-nested-pipeline - inputs: - parameters: + value: '{{tasks.mantle-driver.outputs.parameters.condition}}' + depends: mantle-driver.Succeeded + name: mantle + template: comp-mantle + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index d4449f46ba2..17a99a50d36 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -85,56 +85,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -152,16 +103,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: parameter-oneof-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -224,7 +199,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -318,54 +293,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -386,15 +314,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: parameter-oneof-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index a7bd6591b66..bb3aa1e635a 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -79,56 +79,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +97,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -218,7 +193,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -278,54 +253,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -346,15 +274,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 7e44c6c92b3..6eff504f425 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -50,56 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +68,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,54 +224,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -317,15 +245,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index dcbf2616693..5e054030b17 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -66,56 +66,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -133,16 +84,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-conditional + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -205,7 +180,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -266,54 +241,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -334,15 +262,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-conditional + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index b3acfced7f6..622fe99f221 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -35,56 +35,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -102,16 +53,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -174,7 +149,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -259,54 +234,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -327,15 +255,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index ba53f4f0c61..e422da3f1db 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -44,498 +44,498 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline-complex + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op2"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"world"}}}},"taskInfo":{"name":"print-op2"}}' - name: container + value: '{{workflow.parameters.implementations-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' - name: task-name + value: print-op2 - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op2-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.print-op2-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: my-pipeline - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.print-op2-driver.outputs.parameters.cached-decision}}' + depends: print-op2-driver.Succeeded + name: print-op2 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-1 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"msg":{"runtimeValue":{"constant":"Bye!"}}}},"taskInfo":{"name":"print-op"}}' + - name: container + value: '{{workflow.parameters.implementations-f6ead56828d6931739aa4610b58f9697c110a2d8723f04e95256df73a90b2348}}' + - name: task-name + value: print-op + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op"}}' - - name: container - value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task-name - value: print-op - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' - depends: print-op-driver.Succeeded - name: print-op - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-1","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-2"}}' - - name: container - value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task-name - value: print-op-2 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-2-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' - depends: print-op-2-driver.Succeeded - name: print-op-2 - template: system-container-executor - inputs: - parameters: + value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + depends: print-op-driver.Succeeded + name: print-op + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-2 + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline-complex + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-condition-1}}' - name: parent-dag-id - metadata: {} - name: comp-for-loop-2 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-3"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"A_a\"]"}}},"taskInfo":{"name":"print-op-3"}}' - - name: container - value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task-name - value: print-op-3 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-3-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op-3-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-3-driver.outputs.parameters.cached-decision}}' - depends: print-op-3-driver.Succeeded - name: print-op-3 - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-4"},"inputs":{"parameters":{"s":{"componentInputParameter":"pipelinechannel--loop-item-param-3","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"taskInfo":{"name":"print-op-4"}}' - - name: container - value: '{{workflow.parameters.implementations-e7a1060777c9ef84e36a4d54f25d3102abbcbd62d4c8bbb5883c3a2cbcfb5c6d}}' - - name: task-name - value: print-op-4 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-4-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op-4-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-4-driver.outputs.parameters.cached-decision}}' - depends: print-op-4-driver.Succeeded - name: print-op-4 - template: system-container-executor - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-1"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"pipelinechannel--print-op1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--print-op1-Output''] + == ''Hello''"}}' + - name: task-name + value: condition-1 + depends: print-op1.Succeeded + name: condition-1-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: comp-for-loop-4 - outputs: {} - - inputs: - parameters: + value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + depends: condition-1-driver.Succeeded + name: condition-1 + template: comp-condition-1 + when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-comp-condition-2}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-2"},"dependentTasks":["print-op1"],"inputs":{"parameters":{"pipelinechannel--print-op1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"print-op1"}}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--print-op1-Output''] + != ''Hello''"}}' + - name: task-name + value: condition-2 + depends: print-op1.Succeeded + name: condition-2-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: my-pipeline - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": - \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": - \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": - \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-2 - inputs: - parameters: + value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + depends: condition-2-driver.Succeeded + name: condition-2 + template: comp-condition-2 + when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1-2"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' + - name: container + value: '{{workflow.parameters.implementations-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' + - name: task-name + value: print-op1 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op1-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + depends: print-op1-driver.Succeeded + name: print-op1 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-inner-pipeline + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-inner-pipeline}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-inner-pipeline"},"inputs":{"parameters":{"msg":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"taskInfo":{"name":"inner-pipeline"}}' + - name: task-name + value: inner-pipeline + name: inner-pipeline-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id + value: '{{tasks.inner-pipeline-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.inner-pipeline-driver.outputs.parameters.condition}}' + depends: inner-pipeline-driver.Succeeded + name: inner-pipeline + template: comp-inner-pipeline + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-for-loop-2 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-for-loop-2}}' - name: iteration-index - metadata: {} - name: comp-for-loop-2-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-2"},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": - \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": - \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": - \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"foo"}}' - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-2-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: + value: '{{inputs.parameters.iteration-index}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"Hello\", + \"world!\"]"}},"taskInfo":{"name":"for-loop-2"}}' + name: iteration-item-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: comp-for-loop-2-for-loop-2-iterator - outputs: {} - parallelism: 2 - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-4}}' - - name: iteration-index - value: '{{inputs.parameters.iteration-index}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-4"},"iteratorPolicy":{"parallelismLimit":4},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": - \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": - \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": - \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' - name: iteration-item-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' - depends: iteration-item-driver.Succeeded - name: iteration-item - template: comp-for-loop-4 - inputs: - parameters: + value: '{{tasks.iteration-item-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.iteration-item-driver.outputs.parameters.condition}}' + depends: iteration-item-driver.Succeeded + name: iteration-item + template: comp-for-loop-2 + inputs: + parameters: + - name: parent-dag-id + - name: iteration-index + metadata: {} + name: comp-for-loop-2-iteration + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-for-loop-2}}' - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"Hello\", + \"world!\"]"}},"taskInfo":{"name":"for-loop-2"}}' + name: iteration-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - name: iteration-index - metadata: {} - name: comp-for-loop-4-iteration - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-for-loop-4}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-for-loop-4"},"iteratorPolicy":{"parallelismLimit":4},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": - \"1\", \"B_b\": \"10\"}, {\"A_a\": \"2\", \"B_b\": \"20\"}, {\"A_a\": - \"3\", \"B_b\": \"30\"}, {\"A_a\": \"4\", \"B_b\": \"40\"}, {\"A_a\": - \"5\", \"B_b\": \"50\"}, {\"A_a\": \"6\", \"B_b\": \"60\"}, {\"A_a\": - \"7\", \"B_b\": \"70\"}, {\"A_a\": \"8\", \"B_b\": \"80\"}, {\"A_a\": - \"9\", \"B_b\": \"90\"}, {\"A_a\": \"10\", \"B_b\": \"100\"}]"}},"taskInfo":{"name":"bar"}}' - name: iteration-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.iteration-driver.outputs.parameters.execution-id}}' - - name: iteration-index - value: '{{item}}' - depends: iteration-driver.Succeeded - name: iteration-iterations - template: comp-for-loop-4-iteration - withSequence: - count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' - inputs: - parameters: + value: '{{item}}' + depends: iteration-driver.Succeeded + name: iteration-iterations + template: comp-for-loop-2-iteration + withSequence: + count: '{{tasks.iteration-driver.outputs.parameters.iteration-count}}' + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-for-loop-2-for-loop-2-iterator + outputs: {} + - dag: + tasks: + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: comp-for-loop-4-for-loop-4-iterator - outputs: {} - parallelism: 4 - - dag: - tasks: - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-2 - template: comp-for-loop-2-for-loop-2-iterator - - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: for-loop-4 - template: comp-for-loop-4-for-loop-4-iterator - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: for-loop-2 + template: comp-for-loop-2-for-loop-2-iterator + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op1"},"inputs":{"parameters":{"msg":{"componentInputParameter":"msg"}}},"taskInfo":{"name":"print-op1"}}' + - name: container + value: '{{workflow.parameters.implementations-49a7b98eaa82c6aa3d7c3b771b3e28115ed08a84f99a772799da35aeba399db6}}' + - name: task-name + value: print-op1 - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hello world"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op1-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.print-op1-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op1-driver.outputs.parameters.cached-decision}}' + depends: print-op1-driver.Succeeded + name: print-op1 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"msg":"Hello"}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index 5159269cac7..564af1fe8a0 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -51,56 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +69,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -190,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -275,54 +250,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -343,15 +271,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index 536bd4bde56..79567bfc282 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -82,56 +82,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -149,16 +100,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -221,7 +196,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -281,54 +256,7 @@ spec: metadata: {} name: comp-for-loop-2-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -349,15 +277,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: math-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index 806dc496243..8a7b4e7694a 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -23,56 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -90,16 +41,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-after + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -162,7 +137,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -275,54 +250,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -343,15 +271,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-after + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 7566416be01..4f483ba110e 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -50,56 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +68,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-datasets + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -274,54 +249,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -342,15 +270,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-datasets + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index aa85493d68b..cb2ac08c1cb 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -22,56 +22,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -89,16 +40,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: one-step-pipeline-with-concat-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -161,7 +136,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -222,54 +197,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -290,15 +218,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: one-step-pipeline-with-concat-placeholder + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index cda3dd6fe16..c5b0bf06b43 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -48,56 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +66,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: single-condition-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +162,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -296,54 +271,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -364,15 +292,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: single-condition-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 69fe2631d10..5341e0e2aae 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -124,56 +124,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -191,16 +142,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -263,7 +238,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -323,54 +298,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -391,15 +319,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index 4758479db3a..75c56b228f8 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -94,56 +94,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -161,16 +112,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-importer + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -233,7 +208,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -317,54 +292,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -385,15 +313,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-importer + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_pipeline_input_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_pipeline_input_custom_training_job.yaml index 7b090d878d6..1c67d863c87 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_pipeline_input_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_pipeline_input_custom_training_job.yaml @@ -68,56 +68,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -135,16 +86,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -207,7 +182,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -267,54 +242,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -335,15 +263,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index 0b58a804317..6233c5d10c5 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -107,56 +107,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -174,16 +125,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -246,7 +221,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -379,54 +354,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -447,15 +375,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index 527b54e5738..a364bc4ef8f 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -35,56 +35,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -102,16 +53,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-env + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -174,7 +149,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -258,54 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -326,15 +254,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-env + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index e0784f8f808..1c3799a3005 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -47,352 +47,352 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-73aa41a182947039e1bae4bb36fe13a58b2239d2987abbba8f4a51fcc3938390}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Task + failed."}}}},"taskInfo":{"name":"fail-op"}}' - name: container + value: '{{workflow.parameters.implementations-73aa41a182947039e1bae4bb36fe13a58b2239d2987abbba8f4a51fcc3938390}}' - name: task-name + value: fail-op - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: fail-op-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: pipeline-with-exit-handler - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' + depends: fail-op-driver.Succeeded + name: fail-op + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelinechannel--message"}}},"taskInfo":{"name":"print-op-2"}}' + - name: container + value: '{{workflow.parameters.implementations-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' + - name: task-name + value: print-op-2 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-2-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: + value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' + depends: print-op-2-driver.Succeeded + name: print-op-2 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-exit-handler-1 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"Exit + handler has worked!"}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' + - name: container + value: '{{workflow.parameters.implementations-18f176747f4732647de818c82c01b4b7352850ea1bacc9c3fe36676a8fca6e05}}' + - name: task-name + value: print-op + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: print-op-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-8444a6bac7a3d81cc54291a13166a74231d98f9a98861815f15a055edde30ed8}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-fail-op"},"inputs":{"parameters":{"message":{"runtimeValue":{"constantValue":{"stringValue":"Task - failed."}}}}},"taskInfo":{"name":"fail-op"}}' - - name: container - value: '{{workflow.parameters.implementations-8444a6bac7a3d81cc54291a13166a74231d98f9a98861815f15a055edde30ed8}}' - - name: task-name - value: fail-op - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: fail-op-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.fail-op-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.fail-op-driver.outputs.parameters.cached-decision}}' - depends: fail-op-driver.Succeeded - name: fail-op - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-print-op-2"},"inputs":{"parameters":{"message":{"componentInputParameter":"pipelineparam--message"}}},"taskInfo":{"name":"print-op-2"}}' - - name: container - value: '{{workflow.parameters.implementations-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' - - name: task-name - value: print-op-2 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-2-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op-2-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-2-driver.outputs.parameters.cached-decision}}' - depends: print-op-2-driver.Succeeded - name: print-op-2 - template: system-container-executor - inputs: - parameters: + value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' + depends: print-op-driver.Succeeded + name: print-op + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: exit-hook-root-print-op + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-exit-handler-1}}' - name: parent-dag-id - metadata: {} - name: comp-exit-handler-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' - - name: task - value: '{"componentRef":{"name":"comp-print-op"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"message":{"runtimeValue":{"constantValue":{"stringValue":"Exit - handler has worked!"}}}}},"taskInfo":{"name":"print-op"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - - name: container - value: '{{workflow.parameters.implementations-f192dae3a3c4616f7637be7d0414bcffbff11a78dc03bf428f05490caa678f8a}}' - - name: task-name - value: print-op - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: print-op-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.print-op-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.print-op-driver.outputs.parameters.cached-decision}}' - depends: print-op-driver.Succeeded - name: print-op - template: system-container-executor - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelinechannel--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}}' + - name: task-name + value: exit-handler-1 + name: exit-handler-1-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: exit-hook-root-print-op - outputs: {} - - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: pipeline-with-exit-handler - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: + value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + depends: exit-handler-1-driver.Succeeded + hooks: + exit: + arguments: parameters: - - name: component - value: '{{workflow.parameters.components-comp-exit-handler-1}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-exit-handler-1"},"inputs":{"parameters":{"pipelineparam--message":{"componentInputParameter":"message"}}},"taskInfo":{"name":"exit-handler-1"}}' - - name: task-name - value: exit-handler-1 - name: exit-handler-1-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' - depends: exit-handler-1-driver.Succeeded - hooks: - exit: - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - template: exit-hook-root-print-op - name: exit-handler-1 - template: comp-exit-handler-1 - inputs: - parameters: + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + template: exit-hook-root-print-op + name: exit-handler-1 + template: comp-exit-handler-1 + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"message":"Hello World!"}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameters":{"message":{"stringValue":"Hello World!"}}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index bcffd301e96..26e5e18e7b0 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -112,56 +112,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -179,16 +130,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-google-types + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -251,7 +226,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -348,54 +323,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -416,15 +344,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-google-types + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_if_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_if_placeholder.yaml index 3dcd5d7dc15..fec94574433 100644 --- a/test_data/compiled-workflows/pipeline_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_if_placeholder.yaml @@ -23,56 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -90,16 +41,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -162,7 +137,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -222,54 +197,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -290,15 +218,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index fc7e839429b..c21612f93a8 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -102,56 +102,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -169,16 +120,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-importer + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -241,7 +216,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -314,54 +289,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -382,15 +310,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-importer + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index 1775248e6d1..4aac9fe9652 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -24,56 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -91,16 +42,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-importer-and-gcpc-type + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -163,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -293,54 +268,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -361,15 +289,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-importer-and-gcpc-type + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index e5a2a1749b3..f5dd1c96b5c 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -10,10 +10,6 @@ spec: value: '{"executorLabel":"exec-echo-state","inputDefinitions":{"parameters":{"status":{"isOptional":true,"parameterType":"TASK_FINAL_STATUS"}}}}' - name: implementations-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo_state"],"command":["sh","-c","\nif - - name: components-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618 - value: '{"executorLabel":"exec-echo-state","inputDefinitions":{"parameters":{"status":{"parameterType":"TASK_FINAL_STATUS"}}}}' - - name: implementations-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","echo_state"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' @@ -30,11 +26,6 @@ spec: value: '{"executorLabel":"exec-some-task"}' - name: implementations-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566 value: '{"args":["--executor_input","{{$}}","--function_to_execute","some_task"],"command":["sh","-c","\nif - echo_state(status: str):\n print(status)\n\n"],"image":"python:3.9"}' - - name: components-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7 - value: '{"executorLabel":"exec-error"}' - - name: implementations-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7 - value: '{"args":["--executor_input","{{$}}","--function_to_execute","error"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps'' @@ -48,11 +39,6 @@ spec: value: '{"dag":{"tasks":{"some-task":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-some-task"},"taskInfo":{"name":"some-task"}}}}}' - name: components-root value: '{"dag":{"tasks":{"echo-state":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}},"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}}}}' - error():\n raise RuntimeError(\"An error\")\n\n"],"image":"python:3.9"}' - - name: components-comp-exit-handler-1 - value: '{"dag":{"tasks":{"error":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-error"},"taskInfo":{"name":"error"}}}}}' - - name: components-root - value: '{"dag":{"tasks":{"echo-state":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}},"exit-handler-1":{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -61,326 +47,326 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: status-state-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-some-task"},"taskInfo":{"name":"some-task"}}' - name: container + value: '{{workflow.parameters.implementations-72cacb6e63f200dc565307f11e5792a0bcc013519ea2a8d43b76245d33942566}}' - name: task-name + value: some-task - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + name: some-task-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.some-task-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: my-pipeline - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.some-task-driver.outputs.parameters.cached-decision}}' + depends: some-task-driver.Succeeded + name: some-task + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-exit-handler-1 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' + - name: container + value: '{{workflow.parameters.implementations-ad5c8365040a29a8273ef9e77f6fa5ace6c5a32f68feaac6a452c8a7c214c21f}}' + - name: task-name + value: echo-state + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: echo-state-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.echo-state-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-error"},"taskInfo":{"name":"error"}}' - - name: container - value: '{{workflow.parameters.implementations-08374905246ef9cc3f314d3e7eccc6bc57caafd6fed83e19fa18046b2b80fac7}}' - - name: task-name - value: error - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: error-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.error-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.error-driver.outputs.parameters.cached-decision}}' - depends: error-driver.Succeeded - name: error - template: system-container-executor - inputs: - parameters: + value: '{{tasks.echo-state-driver.outputs.parameters.cached-decision}}' + depends: echo-state-driver.Succeeded + name: echo-state + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: exit-hook-root-echo-state + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: status-state-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-exit-handler-1}}' - name: parent-dag-id - metadata: {} - name: comp-exit-handler-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo-state"},"dependentTasks":["exit-handler-1"],"inputs":{"parameters":{"status":{"taskFinalStatus":{"producerTask":"exit-handler-1"}}}},"taskInfo":{"name":"echo-state"},"triggerPolicy":{"strategy":"ALL_UPSTREAM_TASKS_COMPLETED"}}' - - name: container - value: '{{workflow.parameters.implementations-5fb0b5a0325f074eb3e5ec2bfc2eb455d5c06dcad9bfd2b12283fc4ae282c618}}' - - name: task-name - value: echo-state - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-state-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.echo-state-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.echo-state-driver.outputs.parameters.cached-decision}}' - depends: echo-state-driver.Succeeded - name: echo-state - template: system-container-executor - inputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}' + - name: task-name + value: exit-handler-1 + name: exit-handler-1-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: exit-hook-root-echo-state - outputs: {} - - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: my-pipeline - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: + value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' + depends: exit-handler-1-driver.Succeeded + hooks: + exit: + arguments: parameters: - - name: component - value: '{{workflow.parameters.components-comp-exit-handler-1}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-exit-handler-1"},"taskInfo":{"name":"exit-handler-1"}}' - - name: task-name - value: exit-handler-1 - name: exit-handler-1-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.exit-handler-1-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.exit-handler-1-driver.outputs.parameters.condition}}' - depends: exit-handler-1-driver.Succeeded - hooks: - exit: - arguments: - parameters: - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - template: exit-hook-root-echo-state - name: exit-handler-1 - template: comp-exit-handler-1 - inputs: - parameters: + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + template: exit-hook-root-echo-state + name: exit-handler-1 + template: comp-exit-handler-1 + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index fab1fe97085..745e1178e59 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -63,56 +63,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -130,16 +81,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -202,7 +177,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -422,54 +397,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -490,15 +418,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-loops + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index f0fb4599321..40675650cea 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -109,56 +109,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -176,16 +127,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-loops-and-conditions-multi-layers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -248,7 +223,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -372,54 +347,7 @@ spec: metadata: {} name: comp-for-loop-7 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -440,15 +368,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-loops-and-conditions-multi-layers + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index 36c9b77f808..967e55ec090 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -60,56 +60,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -127,16 +78,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: dataset-concatenator + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -199,7 +174,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -284,54 +259,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -352,15 +280,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: dataset-concatenator + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index d5fb251888f..3202740090b 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -34,56 +34,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -101,16 +52,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-metrics-outputs + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -173,7 +148,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -233,54 +208,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -301,15 +229,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-metrics-outputs + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index 57b16212c06..d850b73a519 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -53,56 +53,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +71,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-multiple-exit-handlers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -192,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -440,54 +415,7 @@ spec: metadata: {} name: exit-hook-root-print-op-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -508,15 +436,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-multiple-exit-handlers + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 5696ed796dc..496f814166c 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -50,56 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +68,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-conditions-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -274,54 +249,7 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -342,15 +270,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: nested-conditions-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index 3d3c2ffb11a..2fff11b9a0f 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -62,56 +62,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -129,16 +80,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -201,7 +176,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -297,54 +272,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -365,15 +293,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index e24ef559ba4..992ca957f99 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -40,56 +40,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -107,16 +58,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-nested-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -179,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -239,54 +214,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -307,15 +235,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-nested-loops + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index 43e442e28f3..a13aaa85e7f 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -20,56 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -87,16 +38,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: echo-name + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -159,7 +134,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -219,54 +194,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -287,15 +215,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: echo-name + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_ontology.yaml b/test_data/compiled-workflows/pipeline_with_ontology.yaml index 526eec9f498..0da6411bf1c 100644 --- a/test_data/compiled-workflows/pipeline_with_ontology.yaml +++ b/test_data/compiled-workflows/pipeline_with_ontology.yaml @@ -24,56 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - two-step-pipeline-with-ontology - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -91,16 +42,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: two-step-pipeline-with-ontology + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -163,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - two-step-pipeline-with-ontology - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: two-step-pipeline-with-ontology + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index c92c732cc82..c4b845a941f 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -36,56 +36,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -103,16 +54,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -175,7 +150,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -260,54 +235,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -328,15 +256,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index 509a7f16f07..d9ff315d886 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -127,56 +127,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -194,16 +145,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -266,7 +241,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -326,54 +301,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -394,15 +322,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-loops + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 07ce9fe6bdb..c2fe0fba835 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -49,56 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +67,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-pipelineparam-containing-format + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,54 +224,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -317,15 +245,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-pipelineparam-containing-format + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index b709497a5e2..cc4aec68919 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -36,56 +36,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -103,16 +54,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-placeholders + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -175,7 +150,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -235,54 +210,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -303,15 +231,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-placeholders + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index ad87b8e7ac3..47460c83c2d 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -151,56 +151,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -218,16 +169,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -290,7 +265,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -398,7 +373,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -541,7 +516,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -666,7 +641,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -781,7 +756,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -1017,54 +992,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -1085,15 +1013,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_resource_spec.yaml b/test_data/compiled-workflows/pipeline_with_resource_spec.yaml index a93eee047ee..2d09b205948 100644 --- a/test_data/compiled-workflows/pipeline_with_resource_spec.yaml +++ b/test_data/compiled-workflows/pipeline_with_resource_spec.yaml @@ -24,56 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - two-step-pipeline-with-resource-spec - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -91,16 +42,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: two-step-pipeline-with-resource-spec + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -163,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - two-step-pipeline-with-resource-spec - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: two-step-pipeline-with-resource-spec + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index 355236dd080..a74eb2d17bc 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -29,56 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -96,16 +47,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: test-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +159,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -262,54 +237,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,15 +258,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: test-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index 5e7b17c477e..15611fbdc1e 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -21,56 +21,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - add-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -88,16 +39,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: add-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -160,7 +135,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -270,54 +245,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - add-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -338,15 +266,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: add-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index b3254f40cf2..e4a9b215aa6 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -48,317 +48,285 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - inputs: - parameters: + - inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + jsonPath: $.pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + jsonPath: $.cached-decision + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-secret-env + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: - name: component + value: '{{workflow.parameters.components-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab}}' - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["generate-secret-name"],"taskInfo":{"name":"comp"}}' - name: container + value: '{{workflow.parameters.implementations-3d5f1ec2aa4c8fdc12bc567d3a3c13b3a5b868b3420fa4689975e7ba43e403ab}}' - name: task-name + value: comp - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-comp}}' + depends: generate-secret-name.Succeeded + name: comp-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch - valueFrom: - default: "" - jsonPath: $.pod-spec-patch + value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - valueFrom: - default: "false" - jsonPath: $.cached-decision - - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - cached_decision_path: '{{outputs.parameters.cached-decision.path}}' - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - container: '{{inputs.parameters.container}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - http_proxy: "" - https_proxy: "" - iteration_index: '{{inputs.parameters.iteration-index}}' - kubernetes_config: '{{inputs.parameters.kubernetes-config}}' - log_level: "" - no_proxy: "" - pipeline_name: my-pipeline - pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: CONTAINER - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: + value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' + depends: comp-driver.Succeeded + name: comp + template: system-container-executor + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c}}' + - name: task + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-generate-secret-name"},"taskInfo":{"name":"generate-secret-name"}}' + - name: container + value: '{{workflow.parameters.implementations-124566c8aba26be4fd91eef9431ac261402d69907711380549b6e49e0eef904c}}' + - name: task-name + value: generate-secret-name + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: generate-secret-name-driver + template: system-container-driver + - arguments: + parameters: - name: pod-spec-patch + value: '{{tasks.generate-secret-name-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}' - - name: container - value: '{{workflow.parameters.implementations-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' - - name: task-name - value: comp - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-comp}}' - depends: createpvc.Succeeded - name: comp-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}' - depends: comp-driver.Succeeded - name: comp - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}' - - name: container - value: '{{workflow.parameters.implementations-b34273359995b3746ecf1bb58ac4bd6c54d47b6fdc35b013bb7962946f322a19}}' - - name: task-name - value: comp-2 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-comp-2}}' - depends: comp.Succeeded && createpvc.Succeeded - name: comp-2-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.comp-2-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.comp-2-driver.outputs.parameters.cached-decision}}' - depends: comp-2-driver.Succeeded - name: comp-2 - template: system-container-executor - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}' - - name: container - value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}' - - name: task-name - value: createpvc - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: createpvc - template: system-container-driver - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}' - - name: container - value: '{{workflow.parameters.implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}' - - name: task-name - value: deletepvc - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - depends: comp-2.Succeeded && createpvc.Succeeded - name: deletepvc - template: system-container-driver - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - inputs: - parameters: + value: '{{tasks.generate-secret-name-driver.outputs.parameters.cached-decision}}' + depends: generate-secret-name-driver.Succeeded + name: generate-secret-name + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + jsonPath: $.execution-id + - name: iteration-count + valueFrom: + default: "0" + jsonPath: $.iteration-count + - name: condition + valueFrom: + default: "true" + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-secret-env + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' + - dag: + tasks: + - arguments: + parameters: - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - jsonPath: $.execution-id - - name: iteration-count - valueFrom: - default: "0" - jsonPath: $.iteration-count + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"secret_parm":"test-secret-1"}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - name: condition - valueFrom: - default: "true" - jsonPath: $.condition - plugin: - driver-plugin: - args: - cache_disabled: false - component: '{{inputs.parameters.component}}' - condition_path: '{{outputs.parameters.condition.path}}' - dag_execution_id: '{{inputs.parameters.parent-dag-id}}' - execution_id_path: '{{outputs.parameters.execution-id.path}}' - http_proxy: "" - https_proxy: "" - iteration_count_path: '{{outputs.parameters.iteration-count.path}}' - iteration_index: '{{inputs.parameters.iteration-index}}' - log_level: "" - no_proxy: "" - pipeline_name: my-pipeline - publish_logs: "" - run_display_name: "" - run_id: '{{workflow.uid}}' - run_name: '{{workflow.name}}' - runtime_config: '{{inputs.parameters.runtime-config}}' - task: '{{inputs.parameters.task}}' - task_name: '{{inputs.parameters.task-name}}' - type: '{{inputs.parameters.driver-type}}' - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} status: finishedAt: null startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index 0f25aa648bb..ebc8db818ad 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -36,56 +36,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -103,16 +54,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-secret-volume + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -175,7 +150,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -237,54 +212,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -305,15 +233,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-secret-volume + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index 7fd27a3e0b6..eca342f1519 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -29,56 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -96,16 +47,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -168,7 +143,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -228,54 +203,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -296,15 +224,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 2e251035360..cb925c5512f 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b:int) -\u003e int:\n return a + b\n\n"],"image":"python:3.8","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","memory-limit","accelerator-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["cpu-limit","accelerator-limit","accelerator-type","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -81,56 +81,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -148,16 +99,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -220,7 +195,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -351,15 +326,15 @@ spec: - name: component value: '{{workflow.parameters.components-20464d97e7da47d6f03b2cf0145e5debf3cf286ec6297704a735ec252f7453c3}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","memory-limit","accelerator-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["cpu-limit","accelerator-limit","accelerator-type","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-20464d97e7da47d6f03b2cf0145e5debf3cf286ec6297704a735ec252f7453c3}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-type.Succeeded && memory-limit.Succeeded && accelerator-limit.Succeeded - && cpu-limit.Succeeded + depends: cpu-limit.Succeeded && accelerator-limit.Succeeded && accelerator-type.Succeeded + && memory-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: @@ -378,54 +353,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -446,15 +374,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index 7241cf707dd..78a755a4f00 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -43,56 +43,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -110,16 +61,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -182,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -281,54 +256,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -349,15 +277,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: my-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index d8f59df2c4a..383ff7b9098 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -24,56 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -91,16 +42,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-various-types + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -163,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-various-types + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index 5c5a7291c42..50ad631dc92 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -77,56 +77,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +95,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-volume + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -216,7 +191,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -335,54 +310,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -403,15 +331,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-volume + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index 977a92991ec..c9552a2e5fd 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -77,56 +77,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +95,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-volume-no-cache + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -216,7 +191,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -335,54 +310,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -403,15 +331,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-volume-no-cache + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 289d5ff32d1..18c234e4851 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -53,56 +53,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +71,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-workspace + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -192,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -277,54 +252,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -345,15 +273,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pipeline-with-workspace + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_none_input_value.yaml index 9cc2121a795..ed74114d5d8 100644 --- a/test_data/compiled-workflows/placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_none_input_value.yaml @@ -23,56 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -90,16 +41,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder-supply-none + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -162,7 +137,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -222,54 +197,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -290,15 +218,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder-supply-none + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index fd8cc75fa31..881534a05d3 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -51,56 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +69,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: preprocess + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -190,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -250,54 +225,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -318,15 +246,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: preprocess + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index b4afe1c10f1..2779e8e59ff 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -27,56 +27,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -94,16 +45,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: producer-consumer-param-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -166,7 +141,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -251,54 +226,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -319,15 +247,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: producer-consumer-param-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index ae0eed937ab..bd8c0a42658 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -42,45 +42,44 @@ spec: templates: - container: args: - - --type - - CONTAINER + - --executor_type + - importer + - --task_spec + - '{{inputs.parameters.task}}' + - --component_spec + - '{{inputs.parameters.component}}' + - --importer_spec + - '{{inputs.parameters.importer}}' - --pipeline_name - - namespace/n1/pipeline/hello-world + - make-language-model-pipeline - --run_id - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id + - --parent_dag_id - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" + - --pod_name + - $(KFP_POD_NAME) + - --pod_uid + - $(KFP_POD_UID) + - --mlmd_server_address + - $(METADATA_GRPC_SERVICE_HOST) + - --mlmd_server_port + - $(METADATA_GRPC_SERVICE_PORT) command: - - driver - image: ghcr.io/kubeflow/kfp-driver + - launcher-v2 + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: ghcr.io/kubeflow/kfp-launcher:latest name: "" resources: limits: @@ -90,6 +89,15 @@ spec: cpu: 100m memory: 64Mi inputs: + parameters: + - name: task + - name: component + - name: importer + - name: parent-dag-id + metadata: {} + name: system-importer + outputs: {} + - inputs: parameters: - name: component - name: task @@ -107,132 +115,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - - name: retry-max-count - value: '{{inputs.parameters.retry-max-count}}' - - name: retry-backoff-duration - value: '{{inputs.parameters.retry-backoff-duration}}' - - name: retry-backoff-factor - value: '{{inputs.parameters.retry-backoff-factor}}' - - name: retry-backoff-max-duration - value: '{{inputs.parameters.retry-backoff-max-duration}}' - name: executor - template: retry-system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - - default: "0" - name: retry-max-count - - default: "0" - name: retry-backoff-duration - - default: "0" - name: retry-backoff-factor - - default: "0" - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - - name: retry-max-count - - name: retry-backoff-duration - - name: retry-backoff-factor - - name: retry-backoff-max-duration - metadata: {} - name: retry-system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - retryStrategy: - backoff: - duration: '{{inputs.parameters.retry-backoff-duration}}' - factor: '{{inputs.parameters.retry-backoff-factor}}' - maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' - limit: '{{inputs.parameters.retry-max-count}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: make-language-model-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -290,12 +206,12 @@ spec: - /kfp-launcher/launch command: - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher + image: ghcr.io/kubeflow/kfp-launcher:latest name: kfp-launcher resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -327,57 +243,40 @@ spec: tasks: - arguments: parameters: - - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"retryPolicy":{"backoffFactor":2,"backoffMaxDuration":"3600s","maxRetryCount":2},"taskInfo":{"name":"hello-world"}}' - - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' - - name: task-name - value: hello-world + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-importer"},"inputs":{"parameters":{"uri":{"runtimeValue":{"constant":"gs://ml-pipeline-playground/shakespeare1.txt"}}}},"taskInfo":{"name":"importer"}}' + - name: component + value: '{{workflow.parameters.components-comp-importer}}' + - name: importer + value: '{{workflow.parameters.implementations-comp-importer}}' - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.hello-world-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.hello-world-driver.outputs.parameters.cached-decision}}' - - name: retry-max-count - value: "2" - - name: retry-backoff-factor - value: "2" - - name: retry-backoff-max-duration - value: "3600" - depends: hello-world-driver.Succeeded - name: hello-world - template: retry-system-container-executor + name: importer + template: system-importer - arguments: parameters: - name: component - value: '{{workflow.parameters.components-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + value: '{{workflow.parameters.components-df76e661981c011f500d6b9d0ea8ef578644ed9edefe4199648d9b383ca340f6}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world-2"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-make-language-model"},"dependentTasks":["importer"],"inputs":{"artifacts":{"text_dataset":{"taskOutputArtifact":{"outputArtifactKey":"artifact","producerTask":"importer"}}}},"taskInfo":{"name":"make-language-model"}}' - name: container - value: '{{workflow.parameters.implementations-203fce8adabe0cfa7da54b9d3ff79c772136c926974659b51c378727c7ccdfb7}}' + value: '{{workflow.parameters.implementations-df76e661981c011f500d6b9d0ea8ef578644ed9edefe4199648d9b383ca340f6}}' - name: task-name - value: hello-world-non-retry + value: make-language-model - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: hello-world-non-retry-driver + depends: importer.Succeeded + name: make-language-model-driver template: system-container-driver - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.pod-spec-patch}}' + value: '{{tasks.make-language-model-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.hello-world-non-retry-driver.outputs.parameters.cached-decision}}' - depends: hello-world-non-retry-driver.Succeeded - name: hello-world-non-retry + value: '{{tasks.make-language-model-driver.outputs.parameters.cached-decision}}' + depends: make-language-model-driver.Succeeded + name: make-language-model template: system-container-executor inputs: parameters: @@ -385,54 +284,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - namespace/n1/pipeline/hello-world - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -453,15 +305,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: make-language-model-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: @@ -469,7 +344,7 @@ spec: - name: component value: '{{workflow.parameters.components-root}}' - name: runtime-config - value: '{"parameters":{"text":{"stringValue":"hi there"}}}' + value: '{}' - name: driver-type value: ROOT_DAG name: root-driver diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index b81d4e1df15..ba1db87f2a5 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -48,56 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +66,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: pythonic-artifacts-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +162,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -272,54 +247,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -340,15 +268,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: pythonic-artifacts-test + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index abee815b627..f5bba908fa1 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -49,56 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +67,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: make-and-join-datasets + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: make-and-join-datasets + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index b5068ffe7f6..6340539f00a 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -54,56 +54,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -121,16 +72,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: split-datasets-and-return-first + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -193,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -253,54 +228,7 @@ spec: metadata: {} name: comp-splitter-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,15 +249,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: split-datasets-and-return-first + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index 38b0c3081b3..562df9cac8a 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -43,56 +43,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -110,16 +61,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: ray-integration-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -182,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -242,54 +217,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -310,15 +238,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: ray-integration-test + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/ray_job_integration_compiled.yaml b/test_data/compiled-workflows/ray_job_integration_compiled.yaml index bd35b351d1b..922a67fff46 100644 --- a/test_data/compiled-workflows/ray_job_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_job_integration_compiled.yaml @@ -156,56 +156,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -223,16 +174,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: ray-integration-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -295,7 +270,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -355,54 +330,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -423,15 +351,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: ray-integration-test + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/sequential-v2.yaml b/test_data/compiled-workflows/sequential-v2.yaml index edcbf7c9fe4..11dce62ca72 100644 --- a/test_data/compiled-workflows/sequential-v2.yaml +++ b/test_data/compiled-workflows/sequential-v2.yaml @@ -24,56 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -91,16 +42,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: sequential + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -163,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,54 +223,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -316,15 +244,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: sequential + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index fbbe4206644..e152557e370 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -45,56 +45,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -112,16 +63,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: take-nap-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +159,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -269,54 +244,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -337,15 +265,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: take-nap-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index fbbe4206644..e152557e370 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -45,56 +45,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -112,16 +63,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: take-nap-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +159,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -269,54 +244,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -337,15 +265,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: take-nap-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index b507f3d829b..e0f2030ea95 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -25,56 +25,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - two-step-pipeline-containerized - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -92,16 +43,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: two-step-pipeline-containerized + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -164,7 +139,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,54 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - two-step-pipeline-containerized - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -317,15 +245,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: two-step-pipeline-containerized + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 27682d94452..1815f704412 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -84,56 +84,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -151,16 +102,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: test-data-passing-pipeline-1 + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -223,7 +198,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -333,54 +308,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -401,15 +329,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: test-data-passing-pipeline-1 + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index ac7773d3df4..ec9b83a32ac 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -288,56 +288,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -355,16 +306,40 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "" + no_proxy: "" + pipeline_name: xgboost-sample-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -427,7 +402,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -663,54 +638,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -731,15 +659,38 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: true + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "" + no_proxy: "" + pipeline_name: xgboost-sample-pipeline + publish_logs: "" + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: From 1ee2602e90a9826de1b7ea7fb184895b96c3cf9a Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 30 Sep 2025 21:24:14 +0300 Subject: [PATCH 4/4] - remove redundant regenerate code Signed-off-by: arpechenin --- backend/test/compiler/argo_ginkgo_test.go | 44 +------------------ .../compiler/matchers/workflow_matcher.go | 2 +- 2 files changed, 2 insertions(+), 44 deletions(-) diff --git a/backend/test/compiler/argo_ginkgo_test.go b/backend/test/compiler/argo_ginkgo_test.go index 5cc56df2024..1ed75427198 100644 --- a/backend/test/compiler/argo_ginkgo_test.go +++ b/backend/test/compiler/argo_ginkgo_test.go @@ -15,17 +15,14 @@ package compiler import ( - "flag" "fmt" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - matcher "github.com/kubeflow/pipelines/backend/test/compiler/matchers" "os" "path/filepath" - "sigs.k8s.io/yaml" "strings" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/v2/compiler/argocompiler" + matcher "github.com/kubeflow/pipelines/backend/test/compiler/matchers" workflowutils "github.com/kubeflow/pipelines/backend/test/compiler/utils" . "github.com/kubeflow/pipelines/backend/test/constants" "github.com/kubeflow/pipelines/backend/test/logger" @@ -35,8 +32,6 @@ import ( . "github.com/onsi/gomega" ) -var regenerateAllSkipTests = flag.Bool("regenerate", false, "Regenerate all compiled workflow and skip tests") - var _ = BeforeEach(func() { logger.Log("Initializing proxy config...") proxy.InitializeConfigWithEmptyForTests() @@ -72,26 +67,10 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, WORKFL fileNameWithoutExtension := strings.TrimSuffix(pipelineSpecFileName, fileExtension) compiledWorkflowFileName := fileNameWithoutExtension + ".yaml" compiledWorkflowFilePath := filepath.Join(argoYAMLDir, compiledWorkflowFileName) - - if *regenerateAllSkipTests { - pipelineSpecs, platformSpec := workflowutils.LoadPipelineSpecsFromIR( - pipelineSpecFilePath, - param.compilerOptions.CacheDisabled, - nil, - ) - regeneratedWf := workflowutils.GetCompiledArgoWorkflow(pipelineSpecs, platformSpec, ¶m.compilerOptions) - err := RegenerateSpec(compiledWorkflowFilePath, *regeneratedWf) - if err != nil { - Fail(fmt.Sprintf("Critical error: %v", err)) - } - continue - } It(fmt.Sprintf("When I compile %s pipeline spec, then the compiled yaml should be %s", pipelineSpecFileName, compiledWorkflowFileName), func() { test_utils.CheckIfSkipping(pipelineSpecFileName) - pipelineSpecs, platformSpec := workflowutils.LoadPipelineSpecsFromIR(pipelineSpecFilePath, param.compilerOptions.CacheDisabled, nil) compiledWorkflow := workflowutils.GetCompiledArgoWorkflow(pipelineSpecs, platformSpec, ¶m.compilerOptions) - if *createMissingGoldenFiles || *updateGoldenFiles { configuredWorkflow := workflowutils.ConfigureCacheSettings(compiledWorkflow, true) _, err := os.Stat(compiledWorkflowFilePath) @@ -103,7 +82,6 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, WORKFL } } expectedWorkflow := workflowutils.UnmarshallWorkflowYAML(compiledWorkflowFilePath) - if param.compilerOptions.CacheDisabled { expectedWorkflow = workflowutils.ConfigureCacheSettings(expectedWorkflow, false) } @@ -129,23 +107,3 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, WORKFL }) } }) - -func RegenerateSpec(path string, wf v1alpha1.Workflow) error { - wfYml, _ := yaml.Marshal(wf) - wfYmlStr := string(wfYml) - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - var wfLines []string - for _, line := range strings.Split(wfYmlStr, "\n") { - if !strings.Contains(line, "- --cache_disabled") { - wfLines = append(wfLines, line) - } - } - _, err = file.WriteString(strings.Join(wfLines, "\n")) - if err != nil { - return err - } - return nil -} diff --git a/backend/test/compiler/matchers/workflow_matcher.go b/backend/test/compiler/matchers/workflow_matcher.go index 96a233b7a93..62c77b23cac 100644 --- a/backend/test/compiler/matchers/workflow_matcher.go +++ b/backend/test/compiler/matchers/workflow_matcher.go @@ -129,7 +129,7 @@ func CompareWorkflows(actual *v1alpha1.Workflow, expected *v1alpha1.Workflow) { gomega.Expect(actual.Spec.Templates[index].HTTP).To(gomega.Equal(template.HTTP), "HTTP is not same") gomega.Expect(actual.Spec.Templates[index].Memoize).To(gomega.Equal(template.Memoize), "Memoize is not same") gomega.Expect(actual.Spec.Templates[index].Metadata).To(gomega.Equal(template.Metadata), "Metadata is not same") - // gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") + gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") gomega.Expect(actual.Spec.Templates[index].PriorityClassName).To(gomega.Equal(template.PriorityClassName), "PriorityClassName is not same") gomega.Expect(actual.Spec.Templates[index].Resource).To(gomega.Equal(template.Resource), "Resource is not same") gomega.Expect(actual.Spec.Templates[index].Script).To(gomega.Equal(template.Script), "Script is not same")