WorkerDeployments becoming async in nature. (#647)

_**READ BEFORE MERGING:** All PRs require approval by both Server AND
SDK teams before merging! This is why the number of required approvals
is "2" and not "1"--two reviewers from the same team is NOT sufficient.
If your PR is not approved by someone in BOTH teams, it may be summarily
reverted._

<!-- Describe what has changed in this PR -->
**What changed?**
- Added a monotonically increasing counter which shall be used by the
history service when scheduling workflow/activity tasks.
- Add an enum in the RoutingConfig which shall convey the routing config
propagation status. This shall allow users to have a peek at the exact
stage where an update, scheduled by a worker-versioning write API, has
reached.

<!-- Tell your future self why have you made these changes -->
**Why?**
- The main purpose of this is to resolve out-of-sync history and
matching partitions when it comes to deciding how to route tasks

<!-- Are there any breaking changes on binary or code level? -->
None, just new fields and enums, no new APIs


<!-- If this breaks the Server, please provide the Server PR to merge
right after this PR was merged. -->
**Server PR**
temporalio/temporal#8570

---------

Co-authored-by: Carly de Frondeville <[email protected]>

Author: Shivs11

openapi/openapiv2.json

@@ -14114,9 +14114,24 @@
           "type": "string",
           "format": "date-time",
           "description": "Last time ramping version percentage was changed.\nIf ramping version is changed, this is also updated, even if the percentage stays the same."
+        },
+        "revisionNumber": {
+          "type": "string",
+          "format": "int64",
+          "description": "Monotonically increasing value which is incremented on every mutation \nto any field of this message to achieve eventual consistency between task queues and their partitions."
         }
       }
     },
+    "v1RoutingConfigUpdateState": {
+      "type": "string",
+      "enum": [
+        "ROUTING_CONFIG_UPDATE_STATE_UNSPECIFIED",
+        "ROUTING_CONFIG_UPDATE_STATE_IN_PROGRESS",
+        "ROUTING_CONFIG_UPDATE_STATE_COMPLETED"
+      ],
+      "default": "ROUTING_CONFIG_UPDATE_STATE_UNSPECIFIED",
+      "description": "Indicates whether a change to the Routing Config has been\npropagated to all relevant Task Queues and their partitions.\n\n - ROUTING_CONFIG_UPDATE_STATE_IN_PROGRESS: Update to the RoutingConfig is currently in progress.\n - ROUTING_CONFIG_UPDATE_STATE_COMPLETED: Update to the RoutingConfig has completed successfully."
+    },
     "v1ScanWorkflowExecutionsResponse": {
       "type": "object",
       "properties": {
@@ -16092,6 +16107,10 @@
         "managerIdentity": {
           "type": "string",
           "description": "Identity of the client that has the exclusive right to make changes to this Worker Deployment.\nEmpty by default.\nIf this is set, clients whose identity does not match `manager_identity` will not be able to make changes\nto this Worker Deployment. They can either set their own identity as the manager or unset the field to proceed."
+        },
+        "routingConfigUpdateState": {
+          "$ref": "#/definitions/v1RoutingConfigUpdateState",
+          "description": "Indicates whether the routing_config has been fully propagated to all\nrelevant task queues and their partitions."
         }
       },
       "description": "A Worker Deployment (Deployment, for short) represents all workers serving \na shared set of Task Queues. Typically, a Deployment represents one service or \napplication.\nA Deployment contains multiple Deployment Versions, each representing a different \nversion of workers. (see documentation of WorkerDeploymentVersionInfo)\nDeployment records are created in Temporal server automatically when their\nfirst poller arrives to the server.\nExperimental. Worker Deployments are experimental and might significantly change in the future."
@@ -17143,6 +17162,11 @@
         "versionTransition": {
           "$ref": "#/definitions/v1DeploymentVersionTransition",
           "description": "When present, indicates the workflow is transitioning to a different deployment version\n(which may belong to the same deployment name or another). Can indicate one of the following\ntransitions: unversioned -> versioned, versioned -> versioned\non a different deployment version, or versioned -> unversioned.\nNot applicable to workflows with PINNED behavior.\nWhen a workflow with AUTO_UPGRADE behavior creates a new workflow task, it will automatically\nstart a transition to the task queue's current version if the task queue's current version is\ndifferent from the workflow's current deployment version.\nIf the AUTO_UPGRADE workflow is stuck due to backlogged activity or workflow tasks, those\ntasks will be redirected to the task queue's current version. As soon as a poller from\nthat deployment version is available to receive the task, the workflow will automatically\nstart a transition to that version and continue execution there.\nA version transition can only exist while there is a pending or started workflow task.\nOnce the pending workflow task completes on the transition's target version, the\ntransition completes and the workflow's `behavior`, and `deployment_version` fields are updated per the\nworker's task completion response.\nPending activities will not start new attempts during a transition. Once the transition is\ncompleted, pending activities will start their next attempt on the new version."
+        },
+        "revisionNumber": {
+          "type": "string",
+          "format": "int64",
+          "description": "Monotonic counter reflecting the latest routing decision for this workflow execution.\nUsed for staleness detection between history and matching when dispatching tasks to workers.\nIncremented when a workflow execution routes to a new deployment version, which happens \nwhen a worker of the new deployment version completes a workflow task.\nNote: Pinned tasks and sticky tasks send a value of 0 for this field since these tasks do not\nface the problem of inconsistent dispatching that arises from eventual consistency between \ntask queues and their partitions."
         }
       },
       "description": "Holds all the information about worker versioning for a particular workflow execution.\nExperimental. Versioning info is experimental and might change in the future."

openapi/openapiv3.yaml

@@ -11071,6 +11071,9 @@ components:
             Last time ramping version percentage was changed.
              If ramping version is changed, this is also updated, even if the percentage stays the same.
           format: date-time
+        revisionNumber:
+          type: string
+          description: "Monotonically increasing value which is incremented on every mutation \n to any field of this message to achieve eventual consistency between task queues and their partitions."
     Schedule:
       type: object
       properties:
@@ -13347,6 +13350,16 @@ components:
              Empty by default.
              If this is set, clients whose identity does not match `manager_identity` will not be able to make changes
              to this Worker Deployment. They can either set their own identity as the manager or unset the field to proceed.
+        routingConfigUpdateState:
+          enum:
+            - ROUTING_CONFIG_UPDATE_STATE_UNSPECIFIED
+            - ROUTING_CONFIG_UPDATE_STATE_IN_PROGRESS
+            - ROUTING_CONFIG_UPDATE_STATE_COMPLETED
+          type: string
+          description: |-
+            Indicates whether the routing_config has been fully propagated to all
+             relevant task queues and their partitions.
+          format: enum
       description: "A Worker Deployment (Deployment, for short) represents all workers serving \n a shared set of Task Queues. Typically, a Deployment represents one service or \n application.\n A Deployment contains multiple Deployment Versions, each representing a different \n version of workers. (see documentation of WorkerDeploymentVersionInfo)\n Deployment records are created in Temporal server automatically when their\n first poller arrives to the server.\n Experimental. Worker Deployments are experimental and might significantly change in the future."
     WorkerDeploymentInfo_WorkerDeploymentVersionSummary:
       type: object
@@ -14686,6 +14699,9 @@ components:
              worker's task completion response.
              Pending activities will not start new attempts during a transition. Once the transition is
              completed, pending activities will start their next attempt on the new version.
+        revisionNumber:
+          type: string
+          description: "Monotonic counter reflecting the latest routing decision for this workflow execution.\n Used for staleness detection between history and matching when dispatching tasks to workers.\n Incremented when a workflow execution routes to a new deployment version, which happens \n when a worker of the new deployment version completes a workflow task.\n Note: Pinned tasks and sticky tasks send a value of 0 for this field since these tasks do not\n face the problem of inconsistent dispatching that arises from eventual consistency between \n task queues and their partitions."
       description: |-
         Holds all the information about worker versioning for a particular workflow execution.
          Experimental. Versioning info is experimental and might change in the future.

temporal/api/deployment/v1/message.proto

@@ -201,6 +201,10 @@ message WorkerDeploymentInfo {
     // to this Worker Deployment. They can either set their own identity as the manager or unset the field to proceed.
     string manager_identity = 6;
 
+    // Indicates whether the routing_config has been fully propagated to all
+    // relevant task queues and their partitions.
+    temporal.api.enums.v1.RoutingConfigUpdateState routing_config_update_state = 7;
+
     message WorkerDeploymentVersionSummary {
         // Deprecated. Use `deployment_version`.
         string version = 1 [deprecated = true];
@@ -285,4 +289,7 @@ message RoutingConfig {
     // Last time ramping version percentage was changed.
     // If ramping version is changed, this is also updated, even if the percentage stays the same.
     google.protobuf.Timestamp ramping_version_percentage_changed_time = 6;
+    // Monotonically increasing value which is incremented on every mutation 
+    // to any field of this message to achieve eventual consistency between task queues and their partitions.
+    int64 revision_number = 10;
 }

temporal/api/enums/v1/task_queue.proto

@@ -100,3 +100,13 @@ enum RateLimitSource {
     // The value was set as the system default.
     RATE_LIMIT_SOURCE_SYSTEM = 3;
 }
+
+// Indicates whether a change to the Routing Config has been
+// propagated to all relevant Task Queues and their partitions.
+enum RoutingConfigUpdateState {
+    ROUTING_CONFIG_UPDATE_STATE_UNSPECIFIED = 0;
+    // Update to the RoutingConfig is currently in progress.
+    ROUTING_CONFIG_UPDATE_STATE_IN_PROGRESS = 1;
+    // Update to the RoutingConfig has completed successfully.
+    ROUTING_CONFIG_UPDATE_STATE_COMPLETED = 2;
+}

temporal/api/workflow/v1/message.proto

@@ -208,6 +208,14 @@ message WorkflowExecutionVersioningInfo {
     // Pending activities will not start new attempts during a transition. Once the transition is
     // completed, pending activities will start their next attempt on the new version.
     DeploymentVersionTransition version_transition = 6;
+    // Monotonic counter reflecting the latest routing decision for this workflow execution.
+    // Used for staleness detection between history and matching when dispatching tasks to workers.
+    // Incremented when a workflow execution routes to a new deployment version, which happens 
+    // when a worker of the new deployment version completes a workflow task.
+    // Note: Pinned tasks and sticky tasks send a value of 0 for this field since these tasks do not
+    // face the problem of inconsistent dispatching that arises from eventual consistency between 
+    // task queues and their partitions.
+    int64 revision_number = 8;
 }
 
 // Holds information about ongoing transition of a workflow execution from one deployment to another.