Add logging to exceptions in desperation, remove duplicated test (will restore if found needed), miscellaneous

Author: Ickerday

splunklib/binding.py

@@ -102,7 +102,8 @@ def mask_sensitive_data(data):
     if not isinstance(data, dict):
         try:
             data = json.loads(data)
-        except Exception as ex:
+        except Exception as e:
+            logging.error(e)
             return data
 
     # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type
@@ -1400,14 +1401,16 @@ def request(self, url, message, **kwargs):
             try:
                 response = self.handler(url, message, **kwargs)
                 break
-            except Exception:
+            except Exception as e:
+                logging.error(e)
                 if self.retries <= 0:
                     raise
                 else:
                     time.sleep(self.retryDelay)
                     self.retries -= 1
         response = record(response)
         if 400 <= response.status:
+            logging.error(response)
             raise HTTPError(response)
 
         # Update the cookie with any HTTP request

splunklib/client.py

@@ -621,6 +621,7 @@ def restart(self, timeout=None):
                 if not self.restart_required:
                     return result
             except Exception as e:
+                logging.error(e)
                 sleep(1)
         raise Exception("Operation time out.")
 

splunklib/modularinput/script.py

@@ -13,6 +13,7 @@
 # under the License.
 
 from abc import ABCMeta, abstractmethod
+import logging
 import sys
 import xml.etree.ElementTree as ET
 from urllib.parse import urlsplit
@@ -87,6 +88,7 @@ def run_script(self, args, event_writer, input_stream):
                     self.validate_input(validation_definition)
                     return 0
                 except Exception as e:
+                    logging.error(e)
                     root = ET.Element("error")
                     ET.SubElement(root, "message").text = str(e)
                     event_writer.write_xml_document(root)
@@ -99,6 +101,7 @@ def run_script(self, args, event_writer, input_stream):
             return 1
 
         except Exception as e:
+            logging.error(e)
             event_writer.log_exception(str(e))
             return 1
 

splunklib/searchcommands/search_command.py

@@ -18,6 +18,7 @@
 
 import csv
 import io
+import logging
 import os
 import re
 import sys
@@ -971,6 +972,7 @@ def _read_chunk(istream):
         try:
             header = istream.readline()
         except Exception as error:
+            logging.error(error)
             raise RuntimeError(f"Failed to read transport header: {error}")
 
         if not header:
@@ -988,6 +990,7 @@ def _read_chunk(istream):
         try:
             metadata = istream.read(metadata_length)
         except Exception as error:
+            logging.error(error)
             raise RuntimeError(
                 f"Failed to read metadata of length {metadata_length}: {error}"
             )
@@ -997,6 +1000,7 @@ def _read_chunk(istream):
         try:
             metadata = decoder.decode(ensure_str(metadata))
         except Exception as error:
+            logging.error(error)
             raise RuntimeError(
                 f"Failed to parse metadata of length {metadata_length}: {error}"
             )
@@ -1009,6 +1013,7 @@ def _read_chunk(istream):
             if body_length > 0:
                 body = istream.read(body_length)
         except Exception as error:
+            logging.error(error)
             raise RuntimeError(f"Failed to read body of length {body_length}: {error}")
 
         return metadata, ensure_str(body, errors="replace")

tests/searchcommands/chunked_data_stream.py

@@ -1,7 +1,7 @@
-import collections
 import csv
 import io
 import json
+from collections import Iterable, Iterator
 
 import splunklib.searchcommands.internals
 from splunklib.utils import ensure_binary, ensure_str
@@ -15,7 +15,7 @@ def __init__(self, version, meta, data):
         self.data = csv.DictReader(io.StringIO(data.decode("utf-8")), dialect=dialect)
 
 
-class ChunkedDataStreamIter(collections.abc.Iterator):
+class ChunkedDataStreamIter(Iterator):
     def __init__(self, chunk_stream):
         self.chunk_stream = chunk_stream
 
@@ -29,7 +29,7 @@ def next(self):
             raise StopIteration
 
 
-class ChunkedDataStream(collections.abc.Iterable):
+class ChunkedDataStream(Iterable):
     def __iter__(self):
         return ChunkedDataStreamIter(self)
 

tests/searchcommands/test_apps/eventing_app/bin/eventingcsc.py

@@ -15,15 +15,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os, sys
+import os
+import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 from splunklib.searchcommands import (
-    dispatch,
-    EventingCommand,
     Configuration,
+    EventingCommand,
     Option,
-    validators,
+    dispatch,
 )
 
 

tests/searchcommands/test_apps/eventing_app/default/logging.conf

@@ -0,0 +1,62 @@
+#
+# The format and semantics of this file are described in this article at Python.org:
+#
+#     [Configuration file format](https://docs.python.org/2/library/logging.config.html#configuration-file-format)
+#
+[loggers]
+keys = root, splunklib, SearchCommand
+
+[logger_root]
+# Default: WARNING
+level = DEBUG
+# Default: stderr
+handlers = stderr
+
+[logger_splunklib]
+qualname = splunklib
+# Default: WARNING
+level = NOTSET
+# Default: stderr
+handlers = splunklib
+# Default: 1
+propagate = 0
+
+[logger_SearchCommand]
+qualname = SearchCommand
+# Default: WARNING
+level = NOTSET
+# Default: stderr
+handlers = app
+# Default: 1
+propagate = 0
+
+[handlers]
+# See [logging.handlers](https://docs.python.org/2/library/logging.handlers.html)
+keys = app, splunklib, stderr
+
+[handler_app]
+# Select this handler to log events to app.log
+class = logging.handlers.RotatingFileHandler
+level = NOTSET
+args = ('app.log', 'a', 524288000, 9, 'utf-8', True)
+formatter = searchcommands
+
+[handler_splunklib]
+# Select this handler to log events to splunklib.log
+class = logging.handlers.RotatingFileHandler
+args = ('splunklib.log', 'a', 524288000, 9, 'utf-8', True)
+level = NOTSET
+formatter = searchcommands
+
+[handler_stderr]
+# Select this handler to log events to stderr which splunkd redirects to the associated job's search.log file
+class = logging.StreamHandler
+level = NOTSET
+args = (sys.stderr,)
+formatter = searchcommands
+
+[formatters]
+keys = searchcommands
+
+[formatter_searchcommands]
+format = %(asctime)s, Level=%(levelname)s, Pid=%(process)s, File=%(filename)s, Line=%(lineno)s, %(message)s

tests/searchcommands/test_apps/generating_app/bin/generatingcsc.py

@@ -15,15 +15,16 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os, sys
+import os
+import sys
 import time
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 from splunklib.searchcommands import (
-    dispatch,
-    GeneratingCommand,
     Configuration,
+    GeneratingCommand,
     Option,
+    dispatch,
     validators,
 )
 

tests/searchcommands/test_apps/generating_app/default/logging.conf

@@ -0,0 +1,62 @@
+#
+# The format and semantics of this file are described in this article at Python.org:
+#
+#     [Configuration file format](https://docs.python.org/2/library/logging.config.html#configuration-file-format)
+#
+[loggers]
+keys = root, splunklib, SearchCommand
+
+[logger_root]
+# Default: WARNING
+level = DEBUG
+# Default: stderr
+handlers = stderr
+
+[logger_splunklib]
+qualname = splunklib
+# Default: WARNING
+level = NOTSET
+# Default: stderr
+handlers = splunklib
+# Default: 1
+propagate = 0
+
+[logger_SearchCommand]
+qualname = SearchCommand
+# Default: WARNING
+level = NOTSET
+# Default: stderr
+handlers = app
+# Default: 1
+propagate = 0
+
+[handlers]
+# See [logging.handlers](https://docs.python.org/2/library/logging.handlers.html)
+keys = app, splunklib, stderr
+
+[handler_app]
+# Select this handler to log events to app.log
+class = logging.handlers.RotatingFileHandler
+level = NOTSET
+args = ('app.log', 'a', 524288000, 9, 'utf-8', True)
+formatter = searchcommands
+
+[handler_splunklib]
+# Select this handler to log events to splunklib.log
+class = logging.handlers.RotatingFileHandler
+args = ('splunklib.log', 'a', 524288000, 9, 'utf-8', True)
+level = NOTSET
+formatter = searchcommands
+
+[handler_stderr]
+# Select this handler to log events to stderr which splunkd redirects to the associated job's search.log file
+class = logging.StreamHandler
+level = NOTSET
+args = (sys.stderr,)
+formatter = searchcommands
+
+[formatters]
+keys = searchcommands
+
+[formatter_searchcommands]
+format = %(asctime)s, Level=%(levelname)s, Pid=%(process)s, File=%(filename)s, Line=%(lineno)s, %(message)s

tests/searchcommands/test_apps/reporting_app/default/logging.conf

@@ -0,0 +1,62 @@
+#
+# The format and semantics of this file are described in this article at Python.org:
+#
+#     [Configuration file format](https://docs.python.org/2/library/logging.config.html#configuration-file-format)
+#
+[loggers]
+keys = root, splunklib, SearchCommand
+
+[logger_root]
+# Default: WARNING
+level = DEBUG
+# Default: stderr
+handlers = stderr
+
+[logger_splunklib]
+qualname = splunklib
+# Default: WARNING
+level = NOTSET
+# Default: stderr
+handlers = splunklib
+# Default: 1
+propagate = 0
+
+[logger_SearchCommand]
+qualname = SearchCommand
+# Default: WARNING
+level = NOTSET
+# Default: stderr
+handlers = app
+# Default: 1
+propagate = 0
+
+[handlers]
+# See [logging.handlers](https://docs.python.org/2/library/logging.handlers.html)
+keys = app, splunklib, stderr
+
+[handler_app]
+# Select this handler to log events to app.log
+class = logging.handlers.RotatingFileHandler
+level = NOTSET
+args = ('app.log', 'a', 524288000, 9, 'utf-8', True)
+formatter = searchcommands
+
+[handler_splunklib]
+# Select this handler to log events to splunklib.log
+class = logging.handlers.RotatingFileHandler
+args = ('splunklib.log', 'a', 524288000, 9, 'utf-8', True)
+level = NOTSET
+formatter = searchcommands
+
+[handler_stderr]
+# Select this handler to log events to stderr which splunkd redirects to the associated job's search.log file
+class = logging.StreamHandler
+level = NOTSET
+args = (sys.stderr,)
+formatter = searchcommands
+
+[formatters]
+keys = searchcommands
+
+[formatter_searchcommands]
+format = %(asctime)s, Level=%(levelname)s, Pid=%(process)s, File=%(filename)s, Line=%(lineno)s, %(message)s