Replace timestamps; fix nova metadata calls (#7)

* extend test cases; fix tox

* filter timestamps; fix nova mapping of metadata calls

* no python3.7 check

* fix py2/py3 compatibility for unicode

* fix py37

* toxenv

Author: auhlig

.travis.yml

@@ -7,6 +7,12 @@ matrix:
       env: TOXENV=py27
     - python: 3.5
       env: TOXENV=py35
+    - python: 3.6
+      env: TOXENV=py36
+    - python: 3.7
+      dist: xenial
+      sudo: true
+      env: TOXENV=py37
     - env: TOXENV=pep8
 
 install:

doc/cadf.md

@@ -122,13 +122,17 @@ custom_actions:
 
 #### CADF target type URI
 
-The *target type URI* is a CADF specific representation of the request's target URI consisting of
-a service-specific prefix and the target URI without version or UUID strings. 
+A *target type URI* is a CADF specific representation of the request's target URI consisting of
+the service-specific prefix `service/<service_type>` and the target URI without version, UUIDs and name.
+This middleware relies on a [configuration](../etc/) to identify the parts of the target URI which should be replaced.
 
-In most cases this middleware builds the target type URI by concatenating the `service/<service_type>` prefix and
-all parts of the target URI which do not contain a UUID.
-In case a UUID is found, it's substituted by the singular of the previous part.  
-Should this default behaviour not suffice, writing a [custom strategy](./watcher/target_type_uri_strategy.py) might be required.
+The default replacement strategy determines the target type URI of a request by its path in the following order:  
+1. Check whether any of the given `regex_path_mapping`s applies. Replace and return.
+2. Check whether any of the given keywords appear in the path. 
+   Replace the part following the `keyword` with its singular if its not a `keyword` or `keyword_exclusion`.
+3. Determine target type URI by replacing UUIDs with the singular of the previous part.
+
+The default behaviour can be replaced by implementing a [custom strategy](./watcher/target_type_uri_strategy.py).
 
 Examples:
 ```

requirements.txt

@@ -2,3 +2,4 @@ datadog
 pycadf!=2.0.0,>=1.1.0 # Apache-2.0
 WebOb
 six
+pyyaml

tox.ini

@@ -6,7 +6,7 @@ envlist = py27,py35,pep8
 
 [testenv]
 usedevelop = True
-install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
+install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://raw.githubusercontent.com/openstack/requirements/master/upper-constraints.txt} {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
          BRANCH_NAME=master
          CLIENT_NAME=watcher

watcher/cadf_strategy.py

@@ -227,7 +227,7 @@ def _cadf_action_from_body(self, json_body):
         """
         cadf_action = taxonomy.UNKNOWN
         try:
-            if isinstance(json_body, str) or isinstance(json_body, unicode):
+            if isinstance(json_body, str) or isinstance(json_body, six.string_types):
                 json_body = common.load_json_dict(json_body)
             # the 1st key specifies the action type
             os_action = next(iter(json_body))
@@ -349,6 +349,9 @@ def _determine_target_type_uri_by_parts(self, path_parts):
                 # ensure no versions or uids are added to the target_type_uri even if the path starts with one
                 if common.is_version_string(part) or common.is_uid_string(part):
                     continue
+                elif common.is_timestamp_string(part):
+                    target_type_uri.append('version')
+                    continue
                 if len(part) > 1:
                     target_type_uri.append(part)
 

watcher/common.py

@@ -21,6 +21,7 @@
 
 UID_REGEX = '[a-fA-F0-9-?]{32,36}'
 VERSION_REGEX = '^v(?:\d+\.)?(?:\d+\.)?(\*|\d+)$'
+TIMESTAMP_REGEX = '^\d{4}-\d{2}-\d{2}$'
 
 METHOD_ACTION_MAP = {
     'GET': taxonomy.ACTION_READ,
@@ -200,6 +201,18 @@ def is_version_string(string):
     return False
 
 
+def is_timestamp_string(string):
+    """
+    check if the string is a timestamp
+    :param string: potential timestamp string
+    :return: bool
+    """
+    timestamp_regex = re.compile(TIMESTAMP_REGEX)
+    if timestamp_regex.match(string) or str.lower(string) == 'latest':
+        return True
+    return False
+
+
 def endswith_version(string):
     """
     check whether a string ends with a version

watcher/tests/fixtures/nova-complex.yaml

@@ -1,4 +1,5 @@
 path_keywords:
+  - block-device-mapping
   - consoles
   - entries
   - extensions
@@ -53,4 +54,7 @@ path_keywords:
 keyword_exclusions:
   - configure-project
   - label
-  - disable-log-reason
+  - disable-log-reason
+
+regex_path_mapping:
+  - '\S+/meta-data/block-device-mapping/\S+$': 'version/meta-data/block-device-mapping/block-device-mapping'

watcher/tests/test_common.py

@@ -55,6 +55,22 @@ def test_is_uid_string(self):
                 expected
             )
 
+    def test_is_timestamp_string(self):
+        stimuli = {
+            '7df1acb1a2f2478eadf3f350d3f44c51': False,
+            'v2': False,
+            '2009-04-04': True,
+            '2008-02-01': True,
+            '2016-09-02': True,
+        }
+
+        for stim, expected in six.iteritems(stimuli):
+            is_timestamp = common.is_timestamp_string(stim)
+            self.assertEqual(
+                is_timestamp,
+                expected
+            )
+
     def test_trim_prefix(self):
         self.assertEqual(
             common.trim_prefix('service/storage/object/account/container', 'service/storage/object'),

watcher/tests/test_designate.py

@@ -175,6 +175,12 @@ def test_target_type_uri(self):
                 ),
                 'expected': 'service/dns/root'
             },
+            {
+              'request': fake.create_request(
+                  path='/v2/zones/something.com'
+              ),
+              'expected': 'service/dns/zones/zone'
+            },
         ]
 
         for stim in stimuli:

watcher/tests/test_nova.py

@@ -249,6 +249,42 @@ def test_target_type_uri(self):
                 ),
                 'expected': 'service/compute/root'
             },
+            {
+                'request': fake.create_request(
+                    path='/2009-04-04/meta-data/block-device-mapping'
+                ),
+                'expected': 'service/compute/version/meta-data/block-device-mapping'
+            },
+            {
+                'request': fake.create_request(
+                    path='/2009-04-04/meta-data/block-device-mapping/root'
+                ),
+                'expected': 'service/compute/version/meta-data/block-device-mapping/block-device-mapping'
+            },
+            {
+                'request': fake.create_request(
+                    path='/2012-01-12/meta-data'
+                ),
+                'expected': 'service/compute/version/meta-data'
+            },
+            {
+                'request': fake.create_request(
+                    path='/2012-01-12/meta-data/hostname'
+                ),
+                'expected': 'service/compute/version/meta-data/hostname'
+            },
+            {
+                'request': fake.create_request(
+                    path='/2012-01-12'
+                ),
+                'expected': 'service/compute/version'
+            },
+            {
+                'request': fake.create_request(
+                    path='/latest/meta-data'
+                ),
+                'expected': 'service/compute/version/meta-data'
+            }
         ]
 
         for stim in stimuli: