Introduce new version of continuation token

sdk/core/azure-core/CHANGELOG.md

@@ -1,12 +1,10 @@
 # Release History
 
-## 1.37.1 (Unreleased)
-
-### Features Added
+## 1.38.0 (2026-01-08)
 
 ### Breaking Changes
 
-### Bugs Fixed
+- Changed the continuation token format. Continuation tokens generated by previous versions of azure-core are not compatible with this version.
 
 ### Other Changes
 

sdk/core/azure-core/TROUBLESHOOTING.md

@@ -0,0 +1,40 @@
+# Troubleshooting Azure Core
+
+This document provides solutions to common issues you may encounter when using the Azure Core library.
+
+## Continuation Token Compatibility Issues
+
+### Error: "Continuation token from a previous version is not compatible"
+
+**Symptoms:**
+
+You may encounter an error message like:
+
+```
+ValueError: This continuation token is not compatible with this version of azure-core. It may have been generated by a previous version.
+```
+
+**Cause:**
+
+Starting from azure-core version 1.38.0, the continuation token format was changed. This change was made to improve security and portability. Continuation tokens are opaque strings and their internal format is not guaranteed to be stable across versions.
+
+Continuation tokens generated by previous versions of azure-core are not compatible with version 1.38.0 and later.
+
+**Solution:**
+
+Unfortunately, old continuation tokens cannot be migrated to the new version. You will need to:
+
+1. **Start a new long-running operation**: Instead of using the old continuation token, initiate a new request for your long-running operation.
+
+2. **Check operation status via Azure Portal or CLI**: If you need to check the status of an operation that was started with an old token, you can use the Azure Portal or Azure CLI to check the operation status directly.
+
+3. **Update or pin your dependencies**: Ensure that any new continuation tokens are generated and consumed using the same version of azure-core (1.38.0 or later).
+
+**Prevention:**
+
+To avoid this issue in the future:
+
+- When upgrading azure-core, ensure that any stored continuation tokens are either consumed before the upgrade or discarded.
+- Design your application to handle the case where a continuation token may become invalid.
+
+For more information, see the [CHANGELOG](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/CHANGELOG.md) for version 1.38.0.

sdk/core/azure-core/azure/core/_version.py

@@ -9,4 +9,4 @@
 # regenerated.
 # --------------------------------------------------------------------------
 
-VERSION = "1.37.1"
+VERSION = "1.38.0"

sdk/core/azure-core/azure/core/polling/_poller.py

@@ -23,14 +23,14 @@
 # IN THE SOFTWARE.
 #
 # --------------------------------------------------------------------------
-import base64
 import logging
 import threading
 import uuid
 from typing import TypeVar, Generic, Any, Callable, Optional, Tuple, List
 from azure.core.exceptions import AzureError
 from azure.core.tracing.decorator import distributed_trace
 from azure.core.tracing.common import with_current_context
+from ._utils import _encode_continuation_token, _decode_continuation_token
 
 
 PollingReturnType_co = TypeVar("PollingReturnType_co", covariant=True)
@@ -162,9 +162,7 @@ def get_continuation_token(self) -> str:
         :rtype: str
         :return: An opaque continuation token
         """
-        import pickle
-
-        return base64.b64encode(pickle.dumps(self._initial_response)).decode("ascii")
+        return _encode_continuation_token(self._initial_response)
 
     @classmethod
     def from_continuation_token(
@@ -182,9 +180,8 @@ def from_continuation_token(
             deserialization_callback = kwargs["deserialization_callback"]
         except KeyError:
             raise ValueError("Need kwarg 'deserialization_callback' to be recreated from continuation_token") from None
-        import pickle
 
-        initial_response = pickle.loads(base64.b64decode(continuation_token))  # nosec
+        initial_response = _decode_continuation_token(continuation_token)
         return None, initial_response, deserialization_callback
 
 

sdk/core/azure-core/azure/core/polling/_utils.py

@@ -0,0 +1,140 @@
+# --------------------------------------------------------------------------
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the ""Software""), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+# --------------------------------------------------------------------------
+"""Shared utilities for polling continuation token serialization."""
+
+import base64
+import binascii
+import json
+from typing import Any, Dict, Mapping
+
+
+# Current continuation token version
+_CONTINUATION_TOKEN_VERSION = 1
+
+# Error message for incompatible continuation tokens from older versions
+_INCOMPATIBLE_TOKEN_ERROR_MESSAGE = (
+    "This continuation token is not compatible with this version of azure-core. "
+    "It may have been generated by a previous version. "
+    "See https://aka.ms/azsdk/python/core/troubleshoot for more information."
+)
+
+# Headers that are needed for LRO rehydration.
+# We use an allowlist approach for security - only include headers we actually need.
+_LRO_HEADERS = frozenset(
+    [
+        "operation-location",
+        # azure-asyncoperation is included only for back compat with mgmt-core<=1.6.0
+        "azure-asyncoperation",
+        "location",
+        "content-type",
+        "retry-after",
+    ]
+)
+
+
+def _filter_sensitive_headers(headers: Mapping[str, str]) -> Dict[str, str]:
+    """Filter headers to only include those needed for LRO rehydration.
+
+    Uses an allowlist approach - only headers required for polling are included.
+
+    :param headers: The headers to filter.
+    :type headers: Mapping[str, str]
+    :return: A new dictionary with only allowed headers.
+    :rtype: dict[str, str]
+    """
+    return {k: v for k, v in headers.items() if k.lower() in _LRO_HEADERS}
+
+
+def _is_pickle_format(data: bytes) -> bool:
+    """Check if the data appears to be in pickle format.
+
+    Pickle protocol markers start with \\x80 followed by a protocol version byte (1-5).
+
+    :param data: The bytes to check.
+    :type data: bytes
+    :return: True if the data appears to be pickled, False otherwise.
+    :rtype: bool
+    """
+    if not data or len(data) < 2:
+        return False
+    # Check for pickle protocol marker (0x80) followed by protocol version 1-5
+    return data[0:1] == b"\x80" and 1 <= data[1] <= 5
+
+
+def _decode_continuation_token(continuation_token: str) -> Dict[str, Any]:
+    """Decode a base64-encoded JSON continuation token.
+
+    :param continuation_token: The base64-encoded continuation token.
+    :type continuation_token: str
+    :return: The decoded JSON data as a dictionary (the "data" field from the token).
+    :rtype: dict
+    :raises ValueError: If the token is invalid or in an unsupported format.
+    """
+    try:
+        decoded_bytes = base64.b64decode(continuation_token)
+        token = json.loads(decoded_bytes.decode("utf-8"))
+    except binascii.Error:
+        # Invalid base64 input
+        raise ValueError("This doesn't look like a continuation token the sdk created.") from None
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        # Check if the data appears to be from an older version
+        if _is_pickle_format(decoded_bytes):
+            raise ValueError(_INCOMPATIBLE_TOKEN_ERROR_MESSAGE) from None
+        raise ValueError("Invalid continuation token format.") from None
+
+    # Validate token schema - must be a dict with a version field
+    if not isinstance(token, dict) or "version" not in token:
+        raise ValueError("Invalid continuation token format.") from None
+
+    # For now, we only support version 1
+    # Future versions can add handling for older versions here if needed
+    if token["version"] != _CONTINUATION_TOKEN_VERSION:
+        raise ValueError(_INCOMPATIBLE_TOKEN_ERROR_MESSAGE) from None
+
+    return token["data"]
+
+
+def _encode_continuation_token(data: Any) -> str:
+    """Encode data as a base64-encoded JSON continuation token.
+
+    The token includes a version field for future compatibility checking.
+
+    :param data: The data to encode. Must be JSON-serializable.
+    :type data: any
+    :return: The base64-encoded JSON string.
+    :rtype: str
+    :raises TypeError: If the data is not JSON-serializable.
+    """
+    token = {
+        "version": _CONTINUATION_TOKEN_VERSION,
+        "data": data,
+    }
+    try:
+        return base64.b64encode(json.dumps(token, separators=(",", ":")).encode("utf-8")).decode("ascii")
+    except (TypeError, ValueError) as err:
+        raise TypeError(
+            "Unable to generate a continuation token for this operation. Payload is not JSON-serializable."
+        ) from err