fix various issues in extracting comments from original easyconfig file and including them again in dumped easyconfig

Author: boegel

easybuild/framework/easyconfig/format/one.py

@@ -31,6 +31,7 @@
 :author: Kenneth Hoste (Ghent University)
 """
 import os
+import pprint
 import re
 import tempfile
 
@@ -150,12 +151,14 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
 
         elif outer:
             # only reformat outer (iterable) values for (too) long lines (or for select parameters)
-            if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 and line_too_long) or forced):
+            if isinstance(param_val, (list, tuple, dict)) and ((len(param_val) > 1 or line_too_long) or forced):
 
-                item_tmpl = INDENT_4SPACES + '%(item)s,%(comment)s\n'
+                item_tmpl = INDENT_4SPACES + '%(item)s,%(inline_comment)s\n'
+
+                start_char, end_char = param_strval[0], param_strval[-1]
 
                 # start with opening character: [, (, {
-                res = '%s\n' % param_strval[0]
+                res = '%s\n' % start_char
 
                 # add items one-by-one, special care for dict values (order of keys, different format for elements)
                 if isinstance(param_val, dict):
@@ -166,18 +169,32 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
                         else:
                             raise EasyBuildError("Missing mandatory key '%s' in %s.", item_key, param_name)
 
-                        comment = self._get_item_comments(param_name, item_val).get(str(item_val), '')
+                        item_comments = self._get_item_comments(param_name, item_val)
+
+                        inline_comment = item_comments.get('inline', '')
+                        item_tmpl_dict = {'inline_comment': inline_comment}
+
+                        for comment in item_comments.get('above', []):
+                            res += INDENT_4SPACES + comment + '\n'
+
                         key_pref = quote_py_str(item_key) + ': '
-                        addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment)
+                        addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(inline_comment)
                         formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen)
-                        res += item_tmpl % {
-                            'comment': comment,
-                            'item': key_pref + formatted_item_val,
-                        }
+                        item_tmpl_dict['item'] = key_pref + formatted_item_val
+
+                        res += item_tmpl % item_tmpl_dict
+
                 else:  # list, tuple
                     for item in param_val:
-                        comment = self._get_item_comments(param_name, item).get(str(item), '')
-                        addlen = addlen + len(INDENT_4SPACES) + len(comment)
+                        item_comments = self._get_item_comments(param_name, item)
+
+                        inline_comment = item_comments.get('inline', '')
+                        item_tmpl_dict = {'inline_comment': inline_comment}
+
+                        for comment in item_comments.get('above', []):
+                            res += INDENT_4SPACES + comment + '\n'
+
+                        addlen = addlen + len(INDENT_4SPACES) + len(inline_comment)
                         # the tuples are really strings here that are constructed from the dependency dicts
                         # so for a plain list of builddependencies param_val is a list of strings here;
                         # and for iterated builddependencies it is a list of lists of strings
@@ -188,14 +205,20 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
                                 for subitem in item]) + ']'
                         else:
                             itemstr = self._reformat_line(param_name, item, addlen=addlen)
+                        item_tmpl_dict['item'] = itemstr
+
+                        res += item_tmpl % item_tmpl_dict
 
-                        res += item_tmpl % {
-                            'comment': comment,
-                            'item': itemstr
-                        }
+                # take into account possible closing comments
+                # see https://github.com/easybuilders/easybuild-framework/issues/3082
+                end_comments = self._get_item_comments(param_name, end_char)
+                for comment in end_comments.get('above', []):
+                    res += INDENT_4SPACES + comment + '\n'
 
-                # end with closing character: ], ), }
-                res += param_strval[-1]
+                # end with closing character (']', ')', '}'), incl. possible inline comment
+                res += end_char
+                if 'inline' in end_comments:
+                    res += end_comments['inline']
 
         else:
             # dependencies are already dumped as strings, so they do not need to be quoted again
@@ -207,9 +230,14 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
     def _get_item_comments(self, key, val):
         """Get per-item comments for specified parameter name/value."""
         item_comments = {}
-        for comment_key, comment_val in self.comments['iter'].get(key, {}).items():
+
+        for comment_key, comment_val in self.comments['iterabove'].get(key, {}).items():
+            if str(val) in comment_key:
+                item_comments['above'] = comment_val
+
+        for comment_key, comment_val in self.comments['iterinline'].get(key, {}).items():
             if str(val) in comment_key:
-                item_comments[str(val)] = comment_val
+                item_comments['inline'] = comment_val
 
         return item_comments
 
@@ -312,64 +340,134 @@ def extract_comments(self, rawtxt):
         Inline comments on items of iterable values are also extracted.
         """
         self.comments = {
-            'above': {},  # comments for a particular parameter definition
+            'above': {},  # comments above a parameter definition
             'header': [],  # header comment lines
             'inline': {},  # inline comments
-            'iter': {},  # (inline) comments on elements of iterable values
-            'tail': [],
+            'iterabove': {},  # comment above elements of iterable values
+            'iterinline': {},  # inline comments on elements of iterable values
+            'tail': [],  # comment at the end of the easyconfig file
          }
 
-        rawlines = rawtxt.split('\n')
+        parsed_ec = self.get_config_dict()
 
-        # extract header first
-        while rawlines and rawlines[0].startswith('#'):
-            self.comments['header'].append(rawlines.pop(0))
+        param_def_regex = re.compile('^([a-z__0-9]+)\s*=')
+        whitespace_regex = re.compile('^\s*$')
 
-        parsed_ec = self.get_config_dict()
+        def clean_part(part):
+            """Helper function to strip off trailing whitespace + trailing quotes."""
+            return part.rstrip().rstrip("'").rstrip('"')
+
+        def split_on_comment_hash(line, param_key):
+            """Helper function to split line on first (actual) comment character '#'."""
+
+            # string representation of easyconfig parameter value,
+            # used to check is supposed comment isn't actual part of the parameter value
+            # (and thus not actually a comment at all)
+            param_strval = str(parsed_ec.get(param_key))
+
+            parts = line.split('#')
+
+            # first part (before first #) is definitely not part of comment
+            before_comment = parts.pop(0)
 
+            # strip out parts that look like a comment but are actually part of a parameter value
+            while parts and ('#' + clean_part(parts[0])) in param_strval:
+                before_comment += '#' + parts.pop(0)
+
+            comment = '#'.join(parts)
+
+            return before_comment, comment.strip()
+
+        def grab_more_comment_lines(lines, indent, param_key):
+            """Grab more comment lines that match specified indent."""
+
+            comment_lines = []
+
+            while lines and (lines[0].startswith(indent + '#') or whitespace_regex.match(lines[0])):
+                line = lines.pop(0)
+                _, actual_comment = split_on_comment_hash(line, param_key)
+                # prefix comment with '#' unless line was empty
+                if line:
+                    actual_comment = '# ' + actual_comment
+                comment_lines.append(actual_comment.strip())
+
+            return comment_lines
+
+        rawlines = rawtxt.split('\n')
+
+        # extract header first (include empty lines too)
+        self.comments['header'] = grab_more_comment_lines(rawlines, '', None)
+
+        last_param_key = None
         while rawlines:
             rawline = rawlines.pop(0)
+
+            # keep track of last parameter definition we ran into
+            res = param_def_regex.match(rawline)
+            if res:
+                key = res.group(1)
+                if key in parsed_ec:
+                    last_param_key = key
+
+            if last_param_key:
+                before_comment, inline_comment = split_on_comment_hash(rawline, last_param_key)
+
+                # short-circuit to next line in case there are no actual comments on this (non-empty) line
+                if before_comment and not inline_comment:
+                    continue
+
+            # lines that start with a hash indicate (start of a block of) comment line(s)
             if rawline.startswith('#'):
-                comment = []
-                # comment could be multi-line
-                while rawline is not None and (rawline.startswith('#') or not rawline):
-                    # drop empty lines (that don't even include a #)
-                    if rawline:
-                        comment.append(rawline)
-                    # grab next line (if more lines are left)
-                    if rawlines:
-                        rawline = rawlines.pop(0)
+                comment = [rawline] + grab_more_comment_lines(rawlines, '', last_param_key)
+
+                if rawlines:
+                    # try to pin comment to parameter definition below it
+                    # don't consume the line yet though, it may also include inline comments...
+                    res = param_def_regex.match(rawlines[0])
+                    if res:
+                        last_param_key = res.group(1)
+                        self.comments['above'][last_param_key] = comment
                     else:
-                        rawline = None
-
-                if rawline is None:
-                    self.comments['tail'] = comment
+                        # if the comment is not above a parameter definition,
+                        # just use the whole next line to determine where the comment belongs...
+                        self.comments['above'][rawlines[0]] = comment
                 else:
-                    key = rawline.split('=', 1)[0].strip()
-                    self.comments['above'][key] = comment
-
-            elif '#' in rawline:  # inline comment
-                comment_key, comment_val = None, None
-                comment = rawline.rsplit('#', 1)[1].strip()
-                # check whether this line is parameter definition;
-                # if not, assume it's a continuation of a multi-line value
-                if re.match(r'^[a-z_]+\s*=', rawline):
-                    comment_key = rawline.split('=', 1)[0].strip()
+                    # if there are no more lines, the comment (block) is at the tail
+                    self.comments['tail'] = comment
+
+            elif '#' in rawline:
+                # if there's a hash character elsewhere in the line (not at the start),
+                # there are a couple of possibilities:
+                # - inline comment for a parameter definition (at the end of a non-empty line)
+                # - indented comment for an item value of an iterable easyconfig parameter (list, dict, ...)
+                # - inline comment for an item value of an iterable easyconfig parameter
+
+                before_comment, comment = split_on_comment_hash(rawline, last_param_key)
+                comment = ('# ' + comment).rstrip()
+
+                # first check whether current line is an easyconfig parameter definition
+                # if so, the comment is an inline comment
+                if param_def_regex.match(before_comment):
+                    self.comments['inline'][last_param_key] = '  ' + comment
+
+                # if there's only whitespace before the comment,
+                # then we have an indented comment, and we need to figure out for what exactly
+                elif whitespace_regex.match(before_comment):
+                    # first consume possible additional comment lines with same indentation
+                    comment = [comment] + grab_more_comment_lines(rawlines, before_comment, last_param_key)
+
+                    before_comment, inline_comment = split_on_comment_hash(rawlines.pop(0), last_param_key)
+                    comment_key = before_comment.rstrip()
+                    self.comments['iterabove'].setdefault(last_param_key, {})[comment_key] = comment
+                    if inline_comment:
+                        inline_comment = ('  # ' + inline_comment).rstrip()
+                        self.comments['iterinline'].setdefault(last_param_key, {})[comment_key] = inline_comment
                 else:
-                    # determine parameter value where the item value on this line is a part of
-                    for key, val in parsed_ec.items():
-                        item_val = re.sub(r',$', r'', rawline.rsplit('#', 1)[0].strip())
-                        if not isinstance(val, string_type) and item_val in str(val):
-                            comment_key, comment_val = key, item_val
-                            break
-
-                # check if hash actually indicated a comment; or is part of the value
-                if comment_key in parsed_ec:
-                    if comment.replace("'", '').replace('"', '') not in str(parsed_ec[comment_key]):
-                        if comment_val:
-                            self.comments['iter'].setdefault(comment_key, {})[comment_val] = '  # ' + comment
-                        else:
-                            self.comments['inline'][comment_key] = '  # ' + comment
+                    # inline comment for item of iterable value
+                    comment_key = before_comment.rstrip()
+                    self.comments['iterinline'].setdefault(last_param_key, {})[comment_key] = '  ' + comment
+
+        self.log.debug("Extracted comments:\n%s", pprint.pformat(self.comments, width=120))
 
 
 def retrieve_blocks_in_spec(spec, only_blocks, silent=False):