Merge pull request #50 from splunk/fross/enumerate-input-kinds

Inputs now iterate over all kinds, dynamically drawn from the server.

Author: Fred Ross

splunklib/client.py

@@ -1633,31 +1633,24 @@ class Input(Entity):
     typed input classes and is also used when the client does not recognize an
     input kind."""
     def __init__(self, service, path, kind=None, **kwargs):
+        # kind can be omitted (in which case it is inferred from the path)
+        # Otherwise, valid values are the paths from data/inputs ("udp",
+        # "monitor", "tcp/raw"), or two special cases: "tcp" (which is "tcp/raw")
+        # and "splunktcp" (which is "tcp/cooked").
         Entity.__init__(self, service, path, **kwargs)
         if kind is None:
-            for kind, kind_path in INPUT_KINDMAP.iteritems():
-                if kind_path in path:
-                    self.kind = kind
-                    break
+            path_segments = path.split('/')
+            i = path_segments.index('inputs')
+            self.kind = '/'.join(path_segments[i+1:-1])
             assert self.kind is not None
         else:
             self.kind = kind
 
-# Directory of known input kinds that maps from input kind to path relative 
-# to data/inputs, eg: inputs of kind 'splunktcp' map to a relative path
-# of 'tcp/cooked' and therefore an endpoint path of 'data/inputs/tcp/cooked'.
-INPUT_KINDMAP = {
-    'ad': "ad",
-    'monitor': "monitor",
-    'registry': "registry",
-    'script': "script",
-    'tcp': "tcp/raw",
-    'splunktcp': "tcp/cooked",
-    'udp': "udp",
-    'win-event-log-collections': "win-event-log-collections", 
-    'win-perfmon': "win-perfmon",
-    'win-wmi-collections': "win-wmi-collections"
-}
+        # Maintain compatibility with older kind labels
+        if self.kind == 'tcp/raw':
+            self.kind = 'tcp'
+        if self.kind == 'tcp/cooked':
+            self.kind = 'splunktcp'
 
 # Inputs is a "kinded" collection, which is a heterogenous collection where
 # each item is tagged with a kind, that provides a single merged view of all
@@ -1670,15 +1663,13 @@ class Inputs(Collection):
 
     def __init__(self, service, kindmap=None):
         Collection.__init__(self, service, PATH_INPUTS, item=Input)
-        self._kindmap = kindmap if kindmap is not None else INPUT_KINDMAP
-        
+
     def __getitem__(self, key):
         if isinstance(key, tuple) and len(key) == 2:
             # Fetch a single kind
             kind, key = key
             try:
-                kind_path = self._kindmap[kind]
-                response = self.get(kind_path + "/" + key)
+                response = self.get(self.kindpath(kind) + "/" + key)
                 entries = self._load_list(response)
                 if len(entries) > 1:
                     raise AmbiguousReferenceException("Found multiple inputs of kind %s named %s." % (kind, key))
@@ -1695,9 +1686,9 @@ def __getitem__(self, key):
             # Iterate over all the kinds looking for matches.
             kind = None
             candidate = None
-            for kind, kind_path in self._kindmap.iteritems():
+            for kind in self.kinds:
                 try:
-                    response = self.get(kind_path + "/" + key)
+                    response = self.get(kind + "/" + key)
                     entries = self._load_list(response)
                     if len(entries) > 1:
                         raise AmbiguousReferenceException("Found multiple inputs of kind %s named %s." % (kind, key))
@@ -1729,9 +1720,9 @@ def __contains__(self, key):
             # Without a kind, we want to minimize the number of round trips to the server, so we
             # reimplement some of the behavior of __getitem__ in order to be able to stop searching
             # on the first hit.
-            for kind, kind_path in self._kindmap.iteritems():
+            for kind in self.kinds:
                 try:
-                    response = self.get(kind_path + "/" + key)
+                    response = self.get(self.kindpath(kind) + "/" + key)
                     entries = self._load_list(response)
                     if len(entries) > 0:
                         return True
@@ -1758,8 +1749,9 @@ def create(self, kind, name, **kwargs):
         :return: The new input.
         """
         kindpath = self.kindpath(kind)
-        self.service.post(kindpath, name=name, **kwargs)
-        return Input(self.service, _path(kindpath, name), kind)
+        self.post(kindpath, name=name, **kwargs)
+        path = _path(self.path + kindpath, name)
+        return Input(self.service, path, kind)
 
     def delete(self, kind, name=None):
         """Removes an input from the collection.
@@ -1779,18 +1771,40 @@ def itemmeta(self, kind):
         content = _load_atom(response, MATCH_ENTRY_CONTENT)
         return _parse_atom_metadata(content)
 
+    def _get_kind_list(self, subpath=[]):
+        kinds = []
+        response = self.get('/'.join(subpath))
+        content = _load_atom_entries(response)
+        for entry in content:
+            this_subpath = subpath + [entry.title]
+            if entry.title == 'all' or this_subpath == ['tcp','ssl']:
+                continue
+            elif 'create' in [x.rel for x in entry.link]:
+                kinds.append('/'.join(subpath + [entry.title]))
+            else:
+                subkinds = self._get_kind_list(subpath + [entry.title])
+                kinds.extend(subkinds)
+        return kinds
+
     @property
-    def kinds(self):
-        """Returns the list of input kinds that this collection may 
-        contain."""
-        return self._kindmap.keys()
+    def kinds(self, subpath=[]):
+        """Returns the list of input kinds that this collection contains."""
+        return self._get_kind_list()
 
     def kindpath(self, kind):
         """Returns a path to the resources for a given input kind.
 
         :param `kind`: The input kind.
         """
-        return self.path + self._kindmap[kind]
+        if kind in self.kinds:
+            return kind
+        # Special cases
+        elif kind == 'tcp':
+            return 'tcp/raw'
+        elif kind == 'splunktcp':
+            return 'tcp/cooked'
+        else:
+            raise ValueError("No such kind on server: %s" % kind)
 
     def list(self, *kinds, **kwargs):
         """Returns a list of inputs that belong to the collection. You can also
@@ -1816,7 +1830,7 @@ def list(self, *kinds, **kwargs):
         :param `kinds`: The input kinds to return (optional).
         """
         if len(kinds) == 0:
-            kinds = self._kindmap.keys()
+            kinds = self.kinds
         if len(kinds) == 1:
             kind = kinds[0]
             logging.debug("Inputs.list taking short circuit branch for single kind.")
@@ -1847,8 +1861,7 @@ def list(self, *kinds, **kwargs):
         for kind in kinds:
             response = None
             try:
-                response = self.service.get(self.kindpath(kind),
-                                            search=search)
+                response = self.get(self.kindpath(kind), search=search)
             except HTTPError as e:
                 if e.status == 404: 
                     continue # No inputs of this kind

tests/test_input.py

@@ -19,16 +19,13 @@
 
 import splunklib.client as client
 
-test_inputs = [{'kind': 'tcp', 'name': '9999', 'host': 'sdk-test'},
-               {'kind': 'udp', 'name': '9999', 'host': 'sdk-test'}]
-
 class TestRead(testlib.TestCase):
     def test_read(self):
         inputs = self.service.inputs
         # count doesn't work on inputs; known problem tested for in
         # test_collection.py. This test will speed up dramatically
         # when that's fixed.
-        for item in inputs.list(count=5): 
+        for item in inputs.list(count=5):
             self.check_entity(item)
             item.refresh()
             self.check_entity(item)
@@ -85,52 +82,74 @@ class TestInput(testlib.TestCase):
     def setUp(self):
         super(TestInput, self).setUp()
         inputs = self.service.inputs
+        test_inputs = [{'kind': 'tcp', 'name': '9999', 'host': 'sdk-test'},
+                       {'kind': 'udp', 'name': '9999', 'host': 'sdk-test'}]
         self._test_entities = {}
-        for test_input in test_inputs:
-            if (test_input['kind'], test_input['name']) not in self.service.inputs:
-                self._test_entities[test_input['kind']] = \
-                    inputs.create(**test_input)
+
+        base_port = 10000
+        while True:
+            if str(base_port) in inputs:
+                base_port += 1
             else:
-                self._test_entities[test_input['kind']] = \
-                    inputs[test_input['kind'], test_input['name']]
+                break
+
+        self._test_entities['tcp'] = \
+            inputs.create('tcp', str(base_port), host='sdk-test')
+        self._test_entities['udp'] = \
+            inputs.create('udp', str(base_port), host='sdk-test')
 
     def tearDown(self):
         super(TestInput, self).tearDown()
-        for test_input in test_inputs:
+        for entity in self._test_entities.itervalues():
             try:
                 self.service.inputs.delete(
-                    kind=test_input['kind'],
-                    name=test_input['name'])
+                    kind=entity.kind,
+                    name=entity.name)
             except KeyError:
                 pass
 
+    def test_list(self):
+        inputs = self.service.inputs
+        input_list = inputs.list()
+        self.assertTrue(len(input_list) > 0)
+        for input in input_list:
+            self.assertTrue(input.name is not None)
+
+    def test_lists_modular_inputs(self):
+        if self.service.splunk_version[0] < 5:
+            return # Modular inputs don't exist prior to 5.0
+        else:
+            inputs = self.service.inputs
+            if ('test2','abcd') not in inputs:
+                inputs.create('test2', 'abcd', field1='boris')
+            input = inputs['test2', 'abcd']
+            self.assertEqual(input.field1, 'boris')
+
+
     def test_create(self):
         inputs = self.service.inputs
-        for test_input in test_inputs:
-            kind, name, host = test_input['kind'], test_input['name'], test_input['host']
-            entity = self._test_entities[kind]
-            entity = inputs[kind, name]
+        for entity in self._test_entities.itervalues():
             self.check_entity(entity)
             self.assertTrue(isinstance(entity, client.Input))
-            self.assertEqual(entity.name, name)
-            self.assertEqual(entity.kind, kind)
-            self.assertEqual(entity.host, host)
+
+    def test_get_kind_list(self):
+        inputs = self.service.inputs
+        kinds = inputs._get_kind_list()
+        self.assertTrue('tcp/raw' in kinds)
 
     def test_read(self):
         inputs = self.service.inputs
-        for test_input in test_inputs:
-            kind, name = test_input['kind'], test_input['name']
-            this_entity = self._test_entities[kind]
+        for this_entity in self._test_entities.itervalues():
+            kind, name = this_entity.kind, this_entity.name
             read_entity = inputs[kind, name]
             self.assertEqual(this_entity.kind, read_entity.kind)
             self.assertEqual(this_entity.name, read_entity.name)
             self.assertEqual(this_entity.host, read_entity.host)
 
     def test_update(self):
         inputs = self.service.inputs
-        for test_input in test_inputs:
-            kind, name = test_input['kind'], test_input['name']
-            entity = inputs[kind, name]
+        for entity in self._test_entities.itervalues():
+            kind, name = entity.kind, entity.name
             kwargs = {'host': 'foo', 'sourcetype': 'bar'}
             entity.update(**kwargs)
             entity.refresh()
@@ -139,10 +158,10 @@ def test_update(self):
 
     def test_delete(self):
         inputs = self.service.inputs
-        remaining = len(test_inputs)-1
-        for test_input in test_inputs:
-            kind, name = test_input['kind'], test_input['name']
-            input_entity = self.service.inputs[kind,name]
+        remaining = len(self._test_entities)-1
+        for input_entity in self._test_entities.itervalues():
+            name = input_entity.name
+            kind = input_entity.kind
             self.assertTrue(name in inputs)
             self.assertTrue((kind,name) in inputs)
             if remaining == 0: