Add Deployment Pre-Release APIs (#479)

* Add Deployment APIs

* Move UpdateDeploymentMemo to deployment dir

* Address feedback & rename deployment_name

* Addressed comments

* Addressed feedback

* improve VersioningOverride comments

Author: ShahabT

openapi/openapiv2.json

@@ -37,7 +37,6 @@ import "google/protobuf/empty.proto";
 import "temporal/api/enums/v1/common.proto";
 import "temporal/api/enums/v1/event_type.proto";
 import "temporal/api/enums/v1/reset.proto";
-import "temporal/api/enums/v1/workflow.proto";
 
 message DataBlob {
     temporal.api.enums.v1.EncodingType encoding_type = 1;
@@ -134,8 +133,8 @@ message WorkerVersionStamp {
     // marker for workflow reset points and the BuildIDs search attribute.
     bool use_versioning = 3;
 
-    // Must be sent if user has set a deployment name (versioning-3).
-    string deployment_name = 4;
+    // Must be sent if user has set a deployment series name (versioning-3).
+    string deployment_series_name = 4;
 
     // Later, may include bundle id that could be used for WASM and/or JS dynamically loadable bundles.
 }
@@ -151,32 +150,12 @@ message WorkerVersionCapabilities {
     // tasks.
     bool use_versioning = 2;
 
-    // Must be sent if user has set a deployment name (versioning-3).
-    string deployment_name = 3;
+    // Must be sent if user has set a deployment series name (versioning-3).
+    string deployment_series_name = 4;
 
     // Later, may include info like "I can process WASM and/or JS bundles"
 }
 
-// Used in both UpdateWorkflowExecutionOptions and StartWorkflowExecution to override
-// the versioning behavior of a specific workflow execution. If set, takes precedence
-// over the sdk-sent Versioning Behavior for the specific Workflow Execution it is set on.
-// To remove the override, call UpdateWorkflowExecutionOptions with a null VersioningBehaviorOverride,
-// and use the FieldMask to indicate that it should be mutated.
-message VersioningBehaviorOverride {
-    // Required.
-    temporal.api.enums.v1.VersioningBehavior behavior = 1;
-
-    // Required if behavior is `PINNED`. Must be null if behavior is `AUTO_UPGRADE`.
-    // Identifies the Build ID and Deployment Name to pin the workflow to.
-    WorkerDeployment worker_deployment = 2;
-}
-
-// Identifies a versioned worker deployment.
-message WorkerDeployment {
-    string deployment_name = 1;
-    string build_id = 2;
-}
-
 // Describes where and how to reset a workflow, used for batch reset currently
 // and may be used for single-workflow reset later.
 message ResetOptions {

openapi/openapiv3.yaml

@@ -0,0 +1,89 @@
+// The MIT License
+//
+// Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+syntax = "proto3";
+
+package temporal.api.deployment.v1;
+
+option go_package = "go.temporal.io/api/deployment/v1;deployment";
+option java_package = "io.temporal.api.deployment.v1";
+option java_multiple_files = true;
+option java_outer_classname = "MessageProto";
+option ruby_package = "Temporalio::Api::Deployment::V1";
+option csharp_namespace = "Temporalio.Api.Deployment.V1";
+
+import "google/protobuf/timestamp.proto";
+
+import "temporal/api/enums/v1/task_queue.proto";
+import "temporal/api/common/v1/message.proto";
+
+// `Deployment` identifies a deployment of Temporal workers. The combination of deployment series
+// name + build ID serves as the identifier. User can use `WorkerDeploymentOptions` in their worker
+// programs to specify these values.
+message Deployment {
+    // Different versions of the same worker service/application are related together by having a
+    // shared series name.
+    // Out of all deployments of a series, one can be designated as the current deployment, which
+    // receives new workflow executions and new tasks of workflows with
+    // `VERSIONING_BEHAVIOR_AUTO_UPGRADE` versioning behavior.
+    string series_name = 1;
+    // Build ID changes with each version of the worker when the worker program code and/or config
+    // changes.
+    string build_id = 2;
+}
+
+// `DeploymentInfo` holds information about a deployment. Deployment information is tracked
+// automatically by server as soon as the first poll from that deployment reaches the server. There
+// can be multiple task queue workers in a single deployment which are listed in this message.
+message DeploymentInfo {
+    Deployment deployment = 1;
+    google.protobuf.Timestamp create_time = 2;
+    repeated TaskQueueInfo task_queue_infos = 3;
+    // A user-defined set of key-values. Can be updated as part of write operations to the
+    // deployment, such as `SetCurrentDeployment`.
+    map<string, temporal.api.common.v1.Payload> metadata = 4;
+    // If this deployment is the current deployment of its deployment series.
+    bool is_current = 5;
+
+    message TaskQueueInfo {
+        string name = 1;
+        temporal.api.enums.v1.TaskQueueType type = 2;
+        // When server saw the first poller for this task queue in this deployment.
+        google.protobuf.Timestamp first_poller_time = 3;
+    }
+}
+
+// Used as part of Deployment write APIs to update metadata attached to a deployment.
+message UpdateDeploymentMetadata {
+    map<string, temporal.api.common.v1.Payload> upsert_entries = 1;
+    // List of keys to remove from the metadata.
+    repeated string remove_entries = 2;
+}
+
+// DeploymentListInfo is an abbreviates set of fields from DeploymentInfo that's returned in
+// ListDeployments.
+message DeploymentListInfo {
+    deployment.v1.Deployment deployment = 1;
+    google.protobuf.Timestamp create_time = 2;
+    // If this deployment is the current deployment of its deployment series.
+    bool is_current = 3;
+}

temporal/api/common/v1/message.proto

@@ -0,0 +1,49 @@
+// The MIT License
+//
+// Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+syntax = "proto3";
+
+package temporal.api.enums.v1;
+
+option go_package = "go.temporal.io/api/enums/v1;enums";
+option java_package = "io.temporal.api.enums.v1";
+option java_multiple_files = true;
+option java_outer_classname = "DeploymentProto";
+option ruby_package = "Temporalio::Api::Enums::V1";
+option csharp_namespace = "Temporalio.Api.Enums.V1";
+
+// Specify the reachability level for a deployment so users can decide if it is time to
+// decommission the deployment.
+enum DeploymentReachability {
+    // Reachability level is not specified.
+    DEPLOYMENT_REACHABILITY_UNSPECIFIED = 0;
+    // The deployment is reachable by new and/or open workflows. The deployment cannot be
+    // decommissioned safely.
+    DEPLOYMENT_REACHABILITY_REACHABLE = 1;
+    // The deployment is not reachable by new or open workflows, but might be still needed by
+    // Queries sent to closed workflows. The deployment can be decommissioned safely if user does
+    // not query closed workflows.
+    DEPLOYMENT_REACHABILITY_CLOSED_WORKFLOWS_ONLY = 2;
+    // The deployment is not reachable by any workflow because all the workflows who needed this
+    // deployment went out of retention period. The deployment can be decommissioned safely.
+    DEPLOYMENT_REACHABILITY_UNREACHABLE = 3;
+}

temporal/api/deployment/v1/message.proto

@@ -140,10 +140,23 @@ enum TimeoutType {
 }
 
 enum VersioningBehavior {
+    // Workflow is unversioned. This is the legacy behavior. An unversioned workflow's task may go
+    // to any worker who is polling for the task queue.
     VERSIONING_BEHAVIOR_UNSPECIFIED = 0;
-    // Workflow should be pinned to the current build ID until manually moved.
+    // Workflow should be pinned to the current deployment until completion. Can still be moved
+    // explicitly via `UpdateWorkflowExecutionOptions` API.
+    // Activities of `PINNED` workflows are sent to the same deployment. Exception to this would be
+    // when the activity task queue workers are not present in the workflows deployment, in which
+    // case, the activity will be sent to the current deployment of its own task queue.
     VERSIONING_BEHAVIOR_PINNED = 1;
-    // Workflow automatically moves to the latest version (default build ID of the task queue) when the next
-    // task is dispatched.
+    // Workflow automatically moves to the current deployment of its task queue when the next
+    // workflow task is dispatched.
+    // Activities of `AUTO_UPGRADE` workflows are sent to the current deployment of the workflow
+    // execution based on the last completed workflow task. Exception to this would be when the
+    // activity task queue workers are not present in the workflows deployment, in which case, the
+    // activity will be sent to the current deployment of its own task queue.
+    // Workflows stuck on a backlogged activity will still auto-upgrade if the default deployment
+    // of their task queue changes, without having to wait for the backlogged activity to complete
+    // on the old deployment.
     VERSIONING_BEHAVIOR_AUTO_UPGRADE = 2;
 }

temporal/api/enums/v1/deployment.proto

@@ -40,6 +40,7 @@ import "temporal/api/enums/v1/failed_cause.proto";
 import "temporal/api/enums/v1/update.proto";
 import "temporal/api/enums/v1/workflow.proto";
 import "temporal/api/common/v1/message.proto";
+import "temporal/api/deployment/v1/message.proto";
 import "temporal/api/failure/v1/message.proto";
 import "temporal/api/taskqueue/v1/message.proto";
 import "temporal/api/update/v1/message.proto";
@@ -255,7 +256,7 @@ message WorkflowTaskCompletedEventAttributes {
 message CompletedDeploymentRedirectInfo {
     // The target deployment of the redirect. Null means a previously versioned workflow just
     // completed a redirect to unversioned workers.
-    temporal.api.common.v1.WorkerDeployment deployment = 1;
+    temporal.api.deployment.v1.Deployment deployment = 1;
     // Versioning behavior sent by SDK for this workflow execution on the new deployment.
     // Must be present if and only if `deployment` is set.
     temporal.api.enums.v1.VersioningBehavior versioning_behavior = 2;
@@ -268,7 +269,7 @@ message CompletedDeploymentRedirectInfo {
 message FailedDeploymentRedirectInfo {
     // The target deployment of the failed redirect attempt. Null means a versioned workflow
     // tried to redirect to unversioned workers.
-    temporal.api.common.v1.WorkerDeployment deployment = 1;
+    temporal.api.deployment.v1.Deployment deployment = 1;
 }
 
 message WorkflowTaskTimedOutEventAttributes {

temporal/api/enums/v1/workflow.proto

@@ -38,6 +38,7 @@ import "google/protobuf/timestamp.proto";
 import "temporal/api/enums/v1/common.proto";
 import "temporal/api/enums/v1/workflow.proto";
 import "temporal/api/common/v1/message.proto";
+import "temporal/api/deployment/v1/message.proto";
 import "temporal/api/failure/v1/message.proto";
 import "temporal/api/taskqueue/v1/message.proto";
 import "temporal/api/sdk/v1/user_metadata.proto";
@@ -107,29 +108,27 @@ message WorkflowExecutionInfo {
 
     message VersioningInfo {
         // Determines how server should treat this execution when workers are upgraded.
-        // When present it means that this workflow is versioned. This is not set until an execution
+        // When present it means that this workflow is versioned. This is set after an execution
         // completes its first workflow task on a versioned worker.
         temporal.api.enums.v1.VersioningBehavior behavior = 1;
         // The worker deployment to which this workflow is assigned currently.
         // Must be present if and only if `behavior` is set.
-        temporal.api.common.v1.WorkerDeployment deployment = 2;
-        // Manual override for execution's versioning behavior. Takes precedence over `behavior`.
-        temporal.api.enums.v1.VersioningBehavior behavior_override = 3;
-        // Used to manually pin the execution to a deployment. Must be set when if and only if
-        // `behavior_override` is PINNED. Takes precedence over `deployment`.
-        temporal.api.common.v1.WorkerDeployment deployment_override = 4;
+        temporal.api.deployment.v1.Deployment deployment = 2;
+        // Present if user has set an execution-specific versioning override. This override takes
+        // precedence over SDK-sent `behavior` and `deployment`.
+        VersioningOverride versioning_override = 3;
         // When present, indicates the workflow is being redirected to a different deployment.
-        // A deployment redirect can only exist during the life time of a pending workflow task.
+        // A redirect can only exist during the lifetime of a pending workflow task.
         // If the pending workflow task completes (at the next WorkflowTaskCompleted event), the
         // redirect is considered complete and the workflow's deployment is updated. If the pending
         // workflow task fails or times out, then the redirect is canceled and workflow remains on
         // the previous deployment.
-        RedirectInfo redirect_info = 5;
+        RedirectInfo redirect_info = 4;
 
         message RedirectInfo {
             // The target deployment of the redirect. Null means a so-far-versioned workflow is
             // being redirected to unversioned workers.
-            temporal.api.common.v1.WorkerDeployment deployment = 1;
+            temporal.api.deployment.v1.Deployment deployment = 1;
             // If present, it means the redirect is initiated by a (safe) manual versioning
             // override. Such override is only applied to the workflow when and if the redirect
             // successfully completes.
@@ -192,7 +191,7 @@ message PendingActivityInfo {
 
     // The deployment this activity was dispatched to most recently. Present only if the activity
     // was dispatched to a versioned worker.
-    temporal.api.common.v1.WorkerDeployment last_started_deployment = 19;
+    temporal.api.deployment.v1.Deployment last_started_deployment = 19;
 }
 
 message PendingChildExecutionInfo {
@@ -363,6 +362,20 @@ message NexusOperationCancellationInfo {
 }
 
 message WorkflowExecutionOptions {
-    // If set, takes precedence over the sdk-sent Versioning Behavior for the specific Workflow Execution it is set on.
-    temporal.api.common.v1.VersioningBehaviorOverride versioning_behavior_override = 1;
-}
+    // If set, takes precedence over the sdk-sent Versioning Behavior for the specific Workflow
+    // Execution it is set on.
+    VersioningOverride versioning_override = 1;
+}
+
+// Used to override the versioning behavior and deployment of a specific workflow execution. If set,
+// takes precedence over the worker-sent values. See `WorkflowExecutionInfo.VersioningInfo` for more
+// information. To remove the override, call `UpdateWorkflowExecutionOptions` with a null
+// `VersioningOverride`, and use the `update_mask` to indicate that it should be mutated.
+message VersioningOverride {
+    // Required.
+    temporal.api.enums.v1.VersioningBehavior behavior = 1;
+
+    // Required if behavior is `PINNED`. Must be null if behavior is `AUTO_UPGRADE`.
+    // Identifies the worker deployment to pin the workflow to.
+    temporal.api.deployment.v1.Deployment deployment = 2;
+}