Merge pull request #37 from configcat/offline

LayzLoad TTL, AutoPoll Interval should be stored in cache

Author: kp-cat

configcatclient/autopollingcachepolicy.py

@@ -1,11 +1,14 @@
 import logging
 import sys
-import datetime
 import time
 from threading import Thread, Event
 from requests import HTTPError
 from requests import Timeout
+from datetime import datetime
+from datetime import timedelta
 
+from . import utils
+from .configfetcher import FetchResponse
 from .readwritelock import ReadWriteLock
 from .interfaces import CachePolicy
 
@@ -26,11 +29,11 @@ def __init__(self, config_fetcher, config_cache, cache_key,
         self._config_cache = config_cache
         self._cache_key = cache_key
         self._poll_interval_seconds = poll_interval_seconds
-        self._max_init_wait_time_seconds = datetime.timedelta(seconds=max_init_wait_time_seconds)
+        self._max_init_wait_time_seconds = timedelta(seconds=max_init_wait_time_seconds)
         self._on_configuration_changed_callback = on_configuration_changed_callback
         self._initialized = False
         self._is_running = False
-        self._start_time = datetime.datetime.utcnow()
+        self._start_time = utils.get_utc_now()
         self._lock = ReadWriteLock()
 
         self.thread = Thread(target=self._run, args=[])
@@ -50,32 +53,40 @@ def _run(self):
 
     def get(self):
         while not self._initialized \
-                and datetime.datetime.utcnow() < self._start_time + self._max_init_wait_time_seconds:
+                and utils.get_utc_now() < self._start_time + self._max_init_wait_time_seconds:
             time.sleep(.500)
 
         try:
             self._lock.acquire_read()
-            return self._config_cache.get(self._cache_key)
+            configuration = self._config_cache.get(self._cache_key)
+            return configuration.get(FetchResponse.CONFIG) if configuration else None
         finally:
             self._lock.release_read()
 
     def force_refresh(self):
         try:
             old_configuration = None
-            force_fetch = False
+            etag = ''
 
             try:
                 self._lock.acquire_read()
                 old_configuration = self._config_cache.get(self._cache_key)
-                force_fetch = not bool(old_configuration)
+                if bool(old_configuration):
+                    etag = old_configuration.get(FetchResponse.ETAG, '')
+                    # Cache isn't expired
+                    utc_now = utils.get_utc_now()
+                    if datetime.utcfromtimestamp(old_configuration.get(FetchResponse.FETCH_TIME, 0) + self._poll_interval_seconds) > utc_now:
+                        self._initialized = True
+                        return
             finally:
                 self._lock.release_read()
 
-            configuration_response = self._config_fetcher.get_configuration_json(force_fetch)
+            configuration_response = self._config_fetcher.get_configuration_json(etag)
 
             if configuration_response.is_fetched():
                 configuration = configuration_response.json()
-                if configuration != old_configuration:
+                if configuration is None or old_configuration is None or \
+                        configuration.get(FetchResponse.CONFIG) != old_configuration.get(FetchResponse.CONFIG):
                     try:
                         self._lock.acquire_write()
                         self._config_cache.set(self._cache_key, configuration)

configcatclient/configfetcher.py

@@ -2,8 +2,10 @@
 import logging
 import sys
 from enum import IntEnum
+from platform import python_version
 
 from .datagovernance import DataGovernance
+from .utils import get_utc_now_seconds_since_epoch
 from .version import CONFIGCATCLIENT_VERSION
 from .constants import *
 
@@ -25,14 +27,23 @@ class RedirectMode(IntEnum):
 
 
 class FetchResponse(object):
-    def __init__(self, response):
+    ETAG = 'etag'
+    FETCH_TIME = 'fetch_time'
+    CONFIG = 'config'
+
+    def __init__(self, response, etag='', fetch_time=None):
         self._response = response
+        self._etag = etag
+        self._fetch_time = fetch_time if fetch_time is not None else get_utc_now_seconds_since_epoch()
 
     def json(self):
         """Returns the json-encoded content of a response, if any.
         :raises ValueError: If the response body does not contain valid json.
         """
-        return self._response.json()
+        json = self._response.json()
+        return {FetchResponse.ETAG: self._etag,
+                FetchResponse.FETCH_TIME: self._fetch_time,
+                FetchResponse.CONFIG: json}
 
     def is_fetched(self):
         """Gets whether a new configuration value was fetched or not.
@@ -55,9 +66,9 @@ def __init__(self, sdk_key, mode, base_url=None, proxies=None, proxy_auth=None,
         self._proxy_auth = proxy_auth
         self._connect_timeout = connect_timeout
         self._read_timeout = read_timeout
-        self._etag = ''
         self._headers = {'User-Agent': 'ConfigCat-Python/' + mode + '-' + CONFIGCATCLIENT_VERSION,
                          'X-ConfigCat-UserAgent': 'ConfigCat-Python/' + mode + '-' + CONFIGCATCLIENT_VERSION,
+                         'X-ConfigCat-PythonVersion': python_version(),
                          'Content-Type': "application/json"}
         if base_url is not None:
             self._base_url_overridden = True
@@ -75,31 +86,31 @@ def get_connect_timeout(self):
     def get_read_timeout(self):
         return self._read_timeout
 
-    def get_configuration_json(self, force_fetch=False, retries=0):
+    def get_configuration_json(self, etag='', retries=0):
         """
         :return: Returns the FetchResponse object contains configuration json Dictionary
         """
         uri = self._base_url + '/' + BASE_PATH + self._sdk_key + BASE_EXTENSION
         headers = self._headers
-        if self._etag and not force_fetch:
-            headers['If-None-Match'] = self._etag
+        if etag:
+            headers['If-None-Match'] = etag
         else:
             headers['If-None-Match'] = None
 
         response = requests.get(uri, headers=headers, timeout=(self._connect_timeout, self._read_timeout),
                                 proxies=self._proxies, auth=self._proxy_auth)
         response.raise_for_status()
-        etag = response.headers.get('Etag')
-        if etag:
-            self._etag = etag
+        response_etag = response.headers.get('Etag')
+        if response_etag is None:
+            response_etag = ''
 
-        fetch_response = FetchResponse(response)
+        fetch_response = FetchResponse(response, response_etag)
 
         # If there wasn't a config change, we return the response.
         if not fetch_response.is_fetched():
             return fetch_response
 
-        preferences = fetch_response.json().get(PREFERENCES, None)
+        preferences = fetch_response.json()[FetchResponse.CONFIG].get(PREFERENCES, None)
         if preferences is None:
             return fetch_response
 
@@ -137,4 +148,4 @@ def get_configuration_json(self, force_fetch=False, retries=0):
             return fetch_response
 
         # Retry the config download with the new base_url
-        return self.get_configuration_json(force_fetch, retries + 1)
+        return self.get_configuration_json(etag, retries + 1)

configcatclient/interfaces.py

@@ -22,7 +22,7 @@ def set(self, key, value):
 
 class CachePolicy(object):
     """
-    Config cache interface
+    Cache policy interface
     """
     __metaclass__ = ABCMeta
 

configcatclient/lazyloadingcachepolicy.py

@@ -1,8 +1,12 @@
 import logging
 import sys
-import datetime
+from datetime import datetime
+from datetime import timedelta
+
 from requests import HTTPError
 
+from . import utils
+from .configfetcher import FetchResponse
 from .readwritelock import ReadWriteLock
 from .interfaces import CachePolicy
 
@@ -16,22 +20,23 @@ def __init__(self, config_fetcher, config_cache, cache_key, cache_time_to_live_s
         self._config_fetcher = config_fetcher
         self._config_cache = config_cache
         self._cache_key = cache_key
-        self._cache_time_to_live = datetime.timedelta(seconds=cache_time_to_live_seconds)
+        self._cache_time_to_live = timedelta(seconds=cache_time_to_live_seconds)
         self._lock = ReadWriteLock()
-        self._last_updated = None
 
     def get(self):
-        config = None
+        configuration = None
+        etag = ''
 
         try:
             self._lock.acquire_read()
 
-            config = self._config_cache.get(self._cache_key)
+            configuration = self._config_cache.get(self._cache_key)
+
+            utc_now = utils.get_utc_now()
 
-            utc_now = datetime.datetime.utcnow()
-            if self._last_updated is not None and self._last_updated + self._cache_time_to_live > utc_now:
-                if config is not None:
-                    return config
+            if configuration is not None:
+                if datetime.utcfromtimestamp(configuration.get(FetchResponse.FETCH_TIME, 0)) + self._cache_time_to_live > utc_now:
+                    return configuration.get(FetchResponse.CONFIG)
         finally:
             self._lock.release_read()
 
@@ -40,43 +45,43 @@ def get(self):
             # If while waiting to acquire the write lock another
             # thread has updated the content, then don't bother requesting
             # to the server to minimise time.
-            if config is None or self._last_updated is None or self._last_updated + self._cache_time_to_live <= datetime.datetime.utcnow():
-                force_fetch = not bool(config)
-                self._force_refresh(force_fetch)
+            utc_now = utils.get_utc_now()
+            if configuration is None or datetime.utcfromtimestamp(configuration.get(FetchResponse.FETCH_TIME, 0)) + self._cache_time_to_live <= utc_now:
+                if bool(configuration):
+                    etag = configuration.get(FetchResponse.ETAG, '')
+                self._force_refresh(etag)
         finally:
             self._lock.release_write()
 
         try:
             self._lock.acquire_read()
-            config = self._config_cache.get(self._cache_key)
-            return config
+            configuration = self._config_cache.get(self._cache_key)
+            return configuration.get(FetchResponse.CONFIG) if configuration else None
         finally:
             self._lock.release_read()
 
-    def force_refresh(self):
-        force_fetch = False
-
+    def force_refresh(self, etag=''):
         try:
             self._lock.acquire_read()
-            config = self._config_cache.get(self._cache_key)
-            force_fetch = not bool(config)
+            configuration = self._config_cache.get(self._cache_key)
+            if bool(configuration):
+                etag = configuration.get(FetchResponse.ETAG, '')
         finally:
             self._lock.release_read()
 
         try:
             self._lock.acquire_write()
-            self._force_refresh(force_fetch)
+            self._force_refresh(etag)
         finally:
             self._lock.release_write()
 
-    def _force_refresh(self, force_fetch):
+    def _force_refresh(self, etag):
         try:
-            configuration_response = self._config_fetcher.get_configuration_json(force_fetch)
-            # set _last_updated regardless of whether the cache is updated
+            configuration_response = self._config_fetcher.get_configuration_json(etag)
+            # set _config_cache regardless of whether the cache is updated
             # or whether a 304 not modified has been sent back as the content
             # we have hasn't been updated on the server so not need
             # for subsequent requests to retry this within the cache time to live
-            self._last_updated = datetime.datetime.utcnow()
             if configuration_response.is_fetched():
                 configuration = configuration_response.json()
                 self._config_cache.set(self._cache_key, configuration)

configcatclient/manualpollingcachepolicy.py

@@ -2,6 +2,7 @@
 import sys
 from requests import HTTPError
 
+from .configfetcher import FetchResponse
 from .readwritelock import ReadWriteLock
 from .interfaces import CachePolicy
 
@@ -19,23 +20,24 @@ def get(self):
         try:
             self._lock.acquire_read()
 
-            config = self._config_cache.get(self._cache_key)
-            return config
+            configuration = self._config_cache.get(self._cache_key)
+            return configuration.get(FetchResponse.CONFIG) if configuration else None
         finally:
             self._lock.release_read()
 
     def force_refresh(self):
-        force_fetch = False
+        etag = ''
 
         try:
             self._lock.acquire_read()
-            config = self._config_cache.get(self._cache_key)
-            force_fetch = not bool(config)
+            old_configuration = self._config_cache.get(self._cache_key)
+            if bool(old_configuration):
+                etag = old_configuration.get(FetchResponse.ETAG, '')
         finally:
             self._lock.release_read()
 
         try:
-            configuration_response = self._config_fetcher.get_configuration_json(force_fetch)
+            configuration_response = self._config_fetcher.get_configuration_json(etag)
             if configuration_response.is_fetched():
                 configuration = configuration_response.json()
                 try:

configcatclient/utils.py

@@ -1,6 +1,9 @@
 import sys
 import inspect
 from qualname import qualname
+from datetime import datetime
+
+epoch_time = datetime(1970, 1, 1)
 
 
 def get_class_from_method(method):
@@ -49,3 +52,15 @@ def method_is_called_from(method, level=1):
     if calling_class == expected_class:
         return True
     return False
+
+
+def get_utc_now():
+    return datetime.utcnow()
+
+
+def get_seconds_since_epoch(date_time):
+    return (date_time - epoch_time).total_seconds()
+
+
+def get_utc_now_seconds_since_epoch():
+    return get_seconds_since_epoch(get_utc_now())