Merge pull request hpcugent#340 from stdweird/fix_rest

handle e.g. empty dict body data properly

Author: wdpypere

lib/vsc/utils/generaloption.py

@@ -67,7 +67,7 @@ def set_columns(cols=None):
     if cols is None:
         if os.path.exists(STTY):
             try:
-                proc = subprocess.run(['/usr/bin/stty', 'size'], stdout=subprocess.PIPE)
+                proc = subprocess.run(['/usr/bin/stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
                 cols = str(proc.stdout.splitlines()[0].decode('utf-8').split(' ')[1])
             except (AttributeError, IndexError, OSError, ValueError):
                 # do nothing
@@ -189,12 +189,12 @@ class ExtOption(CompleterOption):
 
     def __init__(self, *args, **kwargs):
         """Add logger to init"""
-        CompleterOption.__init__(self, *args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.log = getLogger(self.__class__.__name__)
 
     def _set_attrs(self, attrs):
         """overwrite _set_attrs to allow store_or callbacks"""
-        Option._set_attrs(self, attrs)
+        super()._set_attrs(attrs)
         if self.action == 'extend':
             # alias
             self.action = 'add'
@@ -240,15 +240,15 @@ def process(self, opt, value, values, parser):
                 self.log.raiseException("%s. Use '%s%s' if the value is correct." % (errmsg, prefix, value),
                                         exception=OptionValueError)
 
-        return Option.process(self, opt, value, values, parser)
+        return super().process(opt, value, values, parser)
 
     def take_action(self, action, dest, opt, value, values, parser):
         """Extended take_action"""
         orig_action = action  # keep copy
 
         # dest is None for actions like shorthelp and confighelp
         if dest and getattr(parser._long_opt.get('--' + dest, ''), 'store_or', '') == 'help':
-            Option.take_action(self, action, dest, opt, value, values, parser)
+            super().take_action(action, dest, opt, value, values, parser)
             fn = getattr(parser, 'print_%shelp' % values.help, None)
             if fn is None:
                 self.log.raiseException("Unsupported output format for help: %s" % value.help, exception=ValueError)
@@ -284,7 +284,7 @@ def take_action(self, action, dest, opt, value, values, parser):
                 if hasattr(values, '_logaction_taken'):
                     values._logaction_taken[dest] = True
 
-            Option.take_action(self, action, dest, opt, value, values, parser)
+            super().take_action(action, dest, opt, value, values, parser)
 
         elif action in self.EXTOPTION_EXTRA_OPTIONS:
             if action in ("add", "add_first", "add_flex",):
@@ -328,7 +328,7 @@ def take_action(self, action, dest, opt, value, values, parser):
                 self.log.raiseException(msg % (action, self.EXTOPTION_EXTRA_OPTIONS))
             setattr(values, dest, lvalue)
         else:
-            Option.take_action(self, action, dest, opt, value, values, parser)
+            super().take_action(action, dest, opt, value, values, parser)
 
         # set flag to mark as passed by action (ie not by default)
         # - distinguish from setting default value through option
@@ -345,12 +345,12 @@ class PassThroughOptionParser(OptionParser):
     from http://www.koders.com/python/fid9DFF5006AF4F52BA6483C4F654E26E6A20DBC73C.aspx?s=add+one#L27
     """
     def __init__(self):
-        OptionParser.__init__(self, add_help_option=False, usage=SUPPRESS_USAGE)
+        super().__init__(add_help_option=False, usage=SUPPRESS_USAGE)
 
     def _process_long_opt(self, rargs, values):
         """Extend optparse code with catch of unknown long options error"""
         try:
-            OptionParser._process_long_opt(self, rargs, values)
+            super()._process_long_opt(rargs, values)
         except BadOptionError as err:
             self.largs.append(err.opt_str)
 
@@ -411,13 +411,14 @@ def __init__(self, *args, **kwargs):
         if section_name in self.RESERVED_SECTIONS:
             self.log.raiseException('Cannot use reserved name %s for section name.' % section_name)
 
-        OptionGroup.__init__(self, *args, **kwargs)
+        super().__init__(*args, **kwargs)
+
         self.section_name = section_name
         self.section_options = []
 
     def add_option(self, *args, **kwargs):
         """Extract configfile section info"""
-        option = OptionGroup.add_option(self, *args, **kwargs)
+        option = super().add_option(*args, **kwargs)
         self.section_options.append(option)
 
         return option
@@ -478,16 +479,9 @@ def __init__(self, *args, **kwargs):
         self.error_env_options = kwargs.pop('error_env_options', False)
         self.error_env_option_method = kwargs.pop('error_env_option_method', self.log.error)
 
-        # py2.4 epilog compatibilty with py2.7 / optparse 1.5.3
-        self.epilog = kwargs.pop('epilog', None)
-
         if 'option_class' not in kwargs:
             kwargs['option_class'] = ExtOption
-        OptionParser.__init__(self, *args, **kwargs)
-
-        # redefine formatter for py2.4 compat
-        if not hasattr(self.formatter, 'format_epilog'):
-            setattr(self.formatter, 'format_epilog', self.formatter.format_description)
+        super().__init__(*args, **kwargs)
 
         if self.epilog is None:
             self.epilog = []
@@ -609,7 +603,7 @@ def format_description(self, formatter):
 
     def set_usage(self, usage):
         """Return usage and set try to set autogenerated description."""
-        OptionParser.set_usage(self, usage)
+        super().set_usage(usage)
 
         if self.description is None:
             self.description = 'NONE_AND_NOT_NONE'
@@ -623,7 +617,7 @@ def get_default_values(self):
                 - only works by using reference to object
                 - same for _logaction_taken
         """
-        values = OptionParser.get_default_values(self)
+        values = super().get_default_values()
 
         class ExtValues(self.VALUES_CLASS):
             _action_taken = {}
@@ -633,19 +627,6 @@ class ExtValues(self.VALUES_CLASS):
         newvalues.__dict__ = values.__dict__.copy()
         return newvalues
 
-    def format_help(self, formatter=None):
-        """For py2.4 compatibility reasons (missing epilog). This is the py2.7 / optparse 1.5.3 code"""
-        if formatter is None:
-            formatter = self.formatter
-        result = []
-        if self.usage:
-            result.append(self.get_usage() + "\n")
-        if self.description:
-            result.append(self.format_description(formatter) + "\n")
-        result.append(self.format_option_help(formatter))
-        result.append(self.format_epilog(formatter))
-        return "".join(result)
-
     def format_epilog(self, formatter):
         """Allow multiple epilog parts"""
         res = []
@@ -694,7 +675,7 @@ def _is_enable_disable(x):
     def print_help(self, fh=None):
         """Intercept print to file to print to string and remove the ENABLE/DISABLE options from help"""
         fh = self.check_help(fh)
-        OptionParser.print_help(self, fh)
+        super().print_help(fh)
 
     def print_rsthelp(self, fh=None):
         """ Print help in rst format """
@@ -797,7 +778,7 @@ def _add_help_option(self):
 
     def _get_args(self, args):
         """Prepend the options set through the environment"""
-        self.commandline_arguments = OptionParser._get_args(self, args)
+        self.commandline_arguments = super()._get_args(args)
         self.get_env_options()
         return self.environment_arguments + self.commandline_arguments  # prepend the environment options as longopts
 

lib/vsc/utils/rest.py

@@ -38,8 +38,8 @@
 import base64
 import copy
 import json
+import logging
 from functools import partial
-from vsc.utils import fancylogger
 from urllib.parse import urlencode
 from urllib.request import Request, HTTPSHandler, build_opener
 
@@ -172,18 +172,19 @@ def request(self, method, url, body, headers, content_type=None):
         secret_items = ['Authorization', 'X-Auth-Token']
         headers_censored = self.censor_request(secret_items, headers)
 
-        if body and not isinstance(body, str):
-            # censor contents of body to avoid leaking passwords
-            secret_items = ['password']
-            body_censored = self.censor_request(secret_items, body)
-            # serialize body in all cases
-            body = json.dumps(body)
-        else:
-            # assume serialized bodies are already clear of secrets
-            fancylogger.getLogger().debug("Request with pre-serialized body, will not censor secrets")
-            body_censored = body
+        body_censored = body
+        if body is not None:
+            if isinstance(body, str):
+                # assume serialized bodies are already clear of secrets
+                logging.debug("Request with pre-serialized body, will not censor secrets")
+            else:
+                # censor contents of body to avoid leaking passwords
+                secret_items = ['password']
+                body_censored = self.censor_request(secret_items, body)
+                # serialize body in all cases
+                body = json.dumps(body)
 
-        fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body_censored, headers_censored)
+        logging.debug('cli request: %s, %s, %s, %s', method, url, body_censored, headers_censored)
 
         # TODO: in recent python: Context manager
         conn = self.get_connection(method, url, body, headers)
@@ -197,7 +198,7 @@ def request(self, method, url, body, headers, content_type=None):
                 pybody = json.loads(body)
             except ValueError:
                 pybody = body
-        fancylogger.getLogger().debug('reponse len: %s ', len(pybody))
+        logging.debug('reponse len: %s ', len(pybody))
         conn.close()
         return status, pybody
 
@@ -241,13 +242,13 @@ def get_connection(self, method, url, body, headers):
             sep = '/'
         else:
             sep = ''
-        if body:
+        if body is not None:
             body = body.encode()
         request = Request(self.url + sep + url, data=body)
         for header, value in headers.items():
             request.add_header(header, value)
         request.get_method = lambda: method
-        fancylogger.getLogger().debug('opening request:  %s%s%s', self.url, sep, url)
+        logging.debug('opening request:  %s%s%s', self.url, sep, url)
         connection = self.opener.open(request)
         return connection
 

setup.py

@@ -37,7 +37,7 @@
 from vsc.install.shared_setup import ag, kh, jt, sdw
 
 PACKAGE = {
-    'version': '3.5.2',
+    'version': '3.5.3',
     'author': [sdw, jt, ag, kh],
     'maintainer': [sdw, jt, ag, kh],
     'install_requires': [