Merge pull request #72 from microsoftgraph/feat/retry-handler

Feat/retry handler

Author: samwelkanda

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+omit =
+    */site-packages/*
+    */distutils/*

.pylintrc

@@ -510,10 +510,10 @@ valid-metaclass-classmethod-first-arg=cls
 [DESIGN]
 
 # Maximum number of arguments for function / method.
-max-args=5
+max-args=7
 
 # Maximum number of attributes for a class (see R0902).
-max-attributes=7
+max-attributes=12
 
 # Maximum number of boolean expressions in an if statement (see R0916).
 max-bool-expr=5

.pypirc

@@ -9,4 +9,4 @@ username = msgraphsdkteam
 
 [testpypi]
 repository = https://test.pypi.org/legacy/
-username = msgraphsdkteam-test
+username = msgraphsdkteam-test

Pipfile.lock

@@ -11,6 +11,7 @@
 from .middleware.abc_token_credential import TokenCredential
 from .middleware.authorization import AuthorizationHandler
 from .middleware.middleware import BaseMiddleware, MiddlewarePipeline
+from .middleware.retry import RetryHandler
 
 
 class HTTPClientFactory:
@@ -51,6 +52,7 @@ def create_with_default_middleware(self, credential: TokenCredential, **kwargs)
         """
         middleware = [
             AuthorizationHandler(credential, **kwargs),
+            RetryHandler(**kwargs),
         ]
         self._register(middleware)
         return self.session

msgraph/core/_client_factory.py

@@ -7,10 +7,26 @@
 from ._client_factory import HTTPClientFactory
 from .middleware.request_context import RequestContext
 
-supported_options = ['scopes', 'custom_option']
+# These are middleware options that can be configured per request.
+# Supports options for default middleware as well as custom middleware.
+supported_options = [
+    # Auth Options
+    'scopes',
+
+    # Retry Options
+    'max_retries',
+    'retry_backoff_factor',
+    'retry_backoff_max',
+    'retry_time_limit',
+    'retry_on_status_codes',
+
+    # Custom middleware options
+    'custom_option',
+]
 
 
 def attach_context(func):
+    """Attaches a request context object to every graph request"""
     def wrapper(*args, **kwargs):
         middleware_control = dict()
 

msgraph/core/_graph_client.py

@@ -0,0 +1,7 @@
+class RetryMiddlewareOptions:
+    def __init__(self, retry_configs):
+        self.retry_total: int = retry_configs.get('retry_total')
+        self.retry_backoff_factor: int = retry_configs.get('retry_backoff_factor')
+        self.retry_backoff_max: int = retry_configs.get('retry_backoff_max')
+        self.retry_time_limit: int = retry_configs.get('retry_time_limit')
+        self.retry_on_status_codes: [int] = retry_configs.get('retry_on_status_codes')

msgraph/core/middleware/options/retry_middleware_options.py

@@ -8,7 +8,22 @@
 
 
 class RequestContext:
+    """A request context contains data that is persisted throughout the request and
+    includes a ClientRequestId property, MiddlewareControl property to control behavior
+    of middleware as well as a FeatureUsage  property to keep track of middleware used
+    in making the request.
+    """
     def __init__(self, middleware_control, headers):
+        """Constructor for request context instances
+
+        Args:
+            middleware_control (dict): A dictionary of optional middleware options
+            that can be accessed by middleware components to override the options provided
+            during middleware initialization,
+
+            headers (dict): A dictionary containing the request headers. Used to check for a
+            user provided client request id.
+        """
         self.middleware_control = middleware_control
         self.client_request_id = headers.get('client-request-id', str(uuid.uuid4()))
         self._feature_usage = FeatureUsageFlag.NONE

msgraph/core/middleware/request_context.py

@@ -0,0 +1,221 @@
+import datetime
+import random
+import sys
+import time
+from email.utils import parsedate_to_datetime
+
+from msgraph.core.middleware.middleware import BaseMiddleware
+
+
+class RetryHandler(BaseMiddleware):
+    """
+    TransportAdapter that allows us to specify the retry policy for all requests
+
+    Retry configuration.
+
+    :param int max_retries:
+        Maximum number of retries to allow. Takes precedence over other counts.
+        Set to ``0`` to fail on the first retry.
+    :param iterable retry_on_status_codes:
+        A set of integer HTTP status codes that we should force a retry on.
+        A retry is initiated if the request method is in ``allowed_methods``
+        and the response status code is in ``RETRY STATUS CODES``.
+    :param float retry_backoff_factor:
+        A backoff factor to apply between attempts after the second try
+        (most errors are resolved immediately by a second try without a
+        delay).
+        The request will sleep for::
+            {backoff factor} * (2 ** ({retry number} - 1))
+        seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
+        for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
+        than :attr:`RetryHandler.MAXIMUM_BACKOFF`.
+        By default, backoff is set to 0.5.
+    :param int retry_time_limit:
+        The maximum cumulative time in seconds that total retries should take.
+        The cumulative retry time and retry-after value for each request retry
+        will be evaluated against this value; if the cumulative retry time plus
+        the retry-after value is greater than the retry_time_limit, the failed
+        response will be immediately returned, else the request retry continues.
+    """
+
+    DEFAULT_MAX_RETRIES = 3
+    MAX_RETRIES = 10
+    DEFAULT_DELAY = 3
+    MAX_DELAY = 180
+    DEFAULT_BACKOFF_FACTOR = 0.5
+    MAXIMUM_BACKOFF = 120
+    _DEFAULT_RETRY_STATUS_CODES = {429, 503, 504}
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.max_retries: int = min(
+            kwargs.pop('max_retries', self.DEFAULT_MAX_RETRIES), self.MAX_RETRIES
+        )
+        self.backoff_factor: float = kwargs.pop('retry_backoff_factor', self.DEFAULT_BACKOFF_FACTOR)
+        self.backoff_max: int = kwargs.pop('retry_backoff_max', self.MAXIMUM_BACKOFF)
+        self.timeout: int = kwargs.pop('retry_time_limit', self.MAX_DELAY)
+
+        status_codes: [int] = kwargs.pop('retry_on_status_codes', [])
+
+        self._retry_on_status_codes: set = set(status_codes) | self._DEFAULT_RETRY_STATUS_CODES
+        self._allowed_methods: set = frozenset(
+            ['HEAD', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']
+        )
+        self._respect_retry_after_header: bool = True
+
+    @classmethod
+    def disable_retries(cls):
+        """
+        Disable retries by setting retry_total to zero.
+        retry_total takes precedence over all other counts.
+        """
+        return cls(max_retries=0)
+
+    def get_retry_options(self, middleware_control):
+        """
+        Check if request specific configs have been passed and override any session defaults
+        Then configure retry settings into the form of a dict.
+        """
+        if middleware_control:
+            return {
+                'total':
+                min(middleware_control.get('max_retries', self.max_retries), self.MAX_RETRIES),
+                'backoff':
+                middleware_control.get('retry_backoff_factor', self.backoff_factor),
+                'max_backoff':
+                middleware_control.get('retry_backoff_max', self.backoff_max),
+                'timeout':
+                middleware_control.get('retry_time_limit', self.timeout),
+                'retry_codes':
+                set(middleware_control.get('retry_on_status_codes', self._retry_on_status_codes))
+                | set(self._DEFAULT_RETRY_STATUS_CODES),
+                'methods':
+                self._allowed_methods,
+            }
+        return {
+            'total': self.max_retries,
+            'backoff': self.backoff_factor,
+            'max_backoff': self.backoff_max,
+            'timeout': self.timeout,
+            'retry_codes': self._retry_on_status_codes,
+            'methods': self._allowed_methods,
+        }
+
+    def send(self, request, **kwargs):
+        """
+        Sends the http request object to the next middleware or retries the request if necessary.
+        """
+        retry_options = self.get_retry_options(request.context.middleware_control)
+        absolute_time_limit = min(retry_options['timeout'], self.MAX_DELAY)
+        response = None
+        retry_count = 0
+        retry_valid = True
+
+        while retry_valid:
+            start_time = time.time()
+            if retry_count > 0:
+                request.headers.update({'retry-attempt': '{}'.format(retry_count)})
+            response = super().send(request, **kwargs)
+            # Check if the request needs to be retried based on the response method
+            # and status code
+            if self.should_retry(retry_options, response):
+                # check that max retries has not been hit
+                retry_valid = self.check_retry_valid(retry_options, retry_count)
+
+                # Get the delay time between retries
+                delay = self.get_delay_time(retry_options, retry_count, response)
+
+                if retry_valid and delay < absolute_time_limit:
+                    time.sleep(delay)
+                    end_time = time.time()
+                    absolute_time_limit -= (end_time - start_time)
+                    # increment the count for retries
+                    retry_count += 1
+
+                    continue
+            break
+        return response
+
+    def should_retry(self, retry_options, response):
+        """
+        Determines whether the request should be retried
+        Checks if the request method is in allowed methods
+        Checks if the response status code is in retryable status codes.
+        """
+        if not self._is_method_retryable(retry_options, response.request):
+            return False
+        if not self._is_request_payload_buffered(response):
+            return False
+        return retry_options['total'] and response.status_code in retry_options['retry_codes']
+
+    def _is_method_retryable(self, retry_options, request):
+        """
+        Checks if a given request should be retried upon, depending on
+        whether the HTTP method is in the set of allowed methods
+        """
+        if request.method.upper() not in retry_options['methods']:
+            return False
+        return True
+
+    def _is_request_payload_buffered(self, response):
+        """
+        Checks if the request payload is buffered/rewindable.
+        Payloads with forward only streams will return false and have the responses
+        returned without any retry attempt.
+        """
+        if response.request.method.upper() in frozenset(['HEAD', 'GET', 'DELETE', 'OPTIONS']):
+            return True
+        if response.request.headers['Content-Type'] == "application/octet-stream":
+            return False
+        return True
+
+    def check_retry_valid(self, retry_options, retry_count):
+        """
+        Check that the max retries limit has not been hit
+        """
+        if retry_count < retry_options['total']:
+            return True
+        return False
+
+    def get_delay_time(self, retry_options, retry_count, response=None):
+        """
+        Get the time in seconds to delay between retry attempts.
+        Respects a retry-after header in the response if provided
+        If no retry-after response header, it defaults to exponential backoff
+        """
+        retry_after = self._get_retry_after(response)
+        if retry_after:
+            return retry_after
+        return self._get_delay_time_exp_backoff(retry_options, retry_count)
+
+    def _get_delay_time_exp_backoff(self, retry_options, retry_count):
+        """
+        Get time in seconds to delay between retry attempts based on an exponential
+        backoff value.
+        """
+        exp_backoff_value = retry_options['backoff'] * +(2**(retry_count - 1))
+        backoff_value = exp_backoff_value + (random.randint(0, 1000) / 1000)
+
+        backoff = min(retry_options['max_backoff'], backoff_value)
+        return backoff
+
+    def _get_retry_after(self, response):
+        """
+        Check if retry-after is specified in the response header and get the value
+        """
+        retry_after = response.headers.get("retry-after")
+        if retry_after:
+            return self._parse_retry_after(retry_after)
+        return None
+
+    def _parse_retry_after(self, retry_after):
+        """
+        Helper to parse Retry-After and get value in seconds.
+        """
+        try:
+            delay = int(retry_after)
+        except ValueError:
+            # Not an integer? Try HTTP date
+            retry_date = parsedate_to_datetime(retry_after)
+            delay = (retry_date - datetime.datetime.now(retry_date.tzinfo)).total_seconds()
+        return max(0, delay)