Address several build reproducibility issues.

The generated cmdlist.py had command list entries written in arbitrary
order based on the order files were found using os.listdir(). The command
list entries are now sorted before writing cmdlist.py.

Further, since the commands list in cmdlist.py is always iterated, its
representation is changed from a dict to a list of tuples. This further
guarantees order, even on pre-Python3.6 interpreters where dict iteration
order is arbitrary.

The fish completions are also made reproducible by replacing some
arbitrarily ordered sets with lists as well as ensuring command aliases are
iterated in a fixed order.

Addresses https://bugs.debian.org/942009

Signed-off-by: Peter Grayson <[email protected]>

Author: jpgrayson

stg-build

@@ -53,7 +53,7 @@ def main():
         argparse.write_asciidoc(stgit.main.commands[options.asciidoc],
                                 sys.stdout)
     elif options.commands:
-        for cmd in sorted(commands.get_commands(allow_cached=False)):
+        for cmd, _, _, _ in commands.get_commands(allow_cached=False):
             print(cmd)
     elif options.cmd_list:
         commands.asciidoc_command_list(

stgit/commands/__init__.py

@@ -9,7 +9,6 @@
 import os
 
 from stgit.compat import text
-from stgit.utils import strip_suffix
 
 __copyright__ = """
 Copyright (C) 2005, Catalin Marinas <[email protected]>
@@ -29,9 +28,9 @@
 """
 
 
-def get_command(mod):
+def get_command(mod_name):
     """Import and return the given command module."""
-    return __import__(__name__ + '.' + mod, globals(), locals(), ['*'])
+    return __import__(__name__ + '.' + mod_name, globals(), locals(), ['*'])
 
 
 _kinds = [('repo', 'Repository commands'),
@@ -46,18 +45,18 @@ def get_command(mod):
 def _find_commands():
     for p in __path__:
         for fn in os.listdir(p):
-            if not fn.endswith('.py'):
+            mod_name, ext = os.path.splitext(fn)
+            if mod_name.startswith('_') or ext != '.py':
                 continue
-            mod = text(strip_suffix('.py', fn))
-            m = get_command(mod)
-            if not hasattr(m, 'usage'):
+            mod = get_command(mod_name)
+            if not hasattr(mod, 'usage'):
                 continue
-            yield mod, m
+            yield mod_name, mod
 
 
 def get_commands(allow_cached=True):
-    """Return a map from command name to a tuple of module name, command
-    type, and one-line command help."""
+    """Return list of tuples of command name, module name, command type, and
+    one-line command help."""
     if allow_cached:
         try:
             from stgit.commands.cmdlist import command_list
@@ -66,25 +65,44 @@ def get_commands(allow_cached=True):
         except ImportError:
             # cmdlist.py doesn't exist, so do it the expensive way.
             pass
-    return dict((text(getattr(m, 'name', mod)), (mod, _kinds[m.kind], m.help))
-                for mod, m in _find_commands())
+    return sorted(
+        (
+            text(getattr(mod, 'name', mod_name)),
+            text(mod_name),
+            _kinds[mod.kind],
+            mod.help,
+        )
+        for mod_name, mod in _find_commands()
+    )
 
 
 def py_commands(commands, f):
-    f.write('from __future__ import unicode_literals\n\n')
-    f.write('command_list = {\n')
-    for name, (mod, kind, help) in commands.items():
-        f.write('    %r: (\n' % name)
-        f.write('        %r,\n' % mod)
-        f.write('        %r,\n' % kind)
-        f.write('        %r,\n' % help)
-        f.write('    ),\n')
-    f.write('}\n')
+    lines = [
+        '# This file is autogenerated.',
+        '',
+        'from __future__ import unicode_literals',
+        '',
+        'command_list = [',
+    ]
+    for cmd_name, mod_name, kind, help in commands:
+        lines.extend(
+            [
+                "    (",
+                "        '%s'," % cmd_name,
+                "        '%s'," % mod_name,
+                "        '%s'," % kind,
+                "        '''%s'''," % help,
+                "    ),",
+            ]
+        )
+    lines.append(']')
+    lines.append('')
+    f.write('\n'.join(lines))
 
 
 def _command_list(commands):
     kinds = {}
-    for cmd, (mod, kind, help) in commands.items():
+    for cmd, mod, kind, help in commands:
         kinds.setdefault(kind, {})[cmd] = help
     for kind in _kind_order:
         kind = _kinds[kind]
@@ -95,7 +113,7 @@ def _command_list(commands):
 
 
 def pretty_command_list(commands, f):
-    cmd_len = max(len(cmd) for cmd in commands)
+    cmd_len = max(len(cmd) for cmd, _, _, _ in commands)
     sep = ''
     for kind, cmds in _command_list(commands):
         f.write(sep)

stgit/completion/bash.py

@@ -224,7 +224,9 @@ def util():
 
 
 def command_list(commands):
-    return ['_stg_commands="%s"\n' % ' '.join(sorted(commands))]
+    return [
+        '_stg_commands="%s"\n' % ' '.join(cmd for cmd, _, _, _ in commands)
+    ]
 
 
 def command_fun(cmd, modname):
@@ -293,7 +295,7 @@ def main_switch(commands):
             'copyright) return ;;'
         ], [
             '%s) _stg_%s ;;' % (cmd, cmd)
-            for cmd in sorted(commands)
+            for cmd, _, _, _ in commands
         ],
         'esac'
     )
@@ -318,7 +320,7 @@ def write_bash_completion(f):
 #    2. Add the following line to your .bashrc:
 #         . ~/.stgit-completion.bash"""]]
     r += [util(), command_list(commands)]
-    for cmd, (modname, _, _) in sorted(commands.items()):
+    for cmd, modname, _, _ in commands:
         r.append(command_fun(cmd, modname))
     r += [main_switch(commands), install()]
     write(f, flatten(r, ''))

stgit/completion/fish.py

@@ -10,23 +10,21 @@
 import stgit.commands
 import stgit.config
 
-_file_args = set(['files', 'dir', 'repo'])
-_has_fish_func_args = set(
-    [
-        'stg_branches',
-        'all_branches',
-        'applied_patches',
-        'unapplied_patches',
-        'hidden_patches',
-        'other_applied_patches',
-        'commit',
-        'conflicting_files',
-        'dirty_files',
-        'unknown_files',
-        'known_files',
-        'mail_aliases',
-    ]
-)
+_file_args = ['files', 'dir', 'repo']
+_has_fish_func_args = [
+    'stg_branches',
+    'all_branches',
+    'applied_patches',
+    'unapplied_patches',
+    'hidden_patches',
+    'other_applied_patches',
+    'commit',
+    'conflicting_files',
+    'dirty_files',
+    'unknown_files',
+    'known_files',
+    'mail_aliases',
+]
 
 
 def _get_file_completion_flag(args):
@@ -60,11 +58,12 @@ def put(*args, **kwargs):
         print(*args, **kwargs)
 
     commands = stgit.commands.get_commands(allow_cached=False)
-    aliases = {}
-    for (name, values) in stgit.config.DEFAULTS.items():
+    aliases = []
+    for name, values in stgit.config.DEFAULTS:
         if name.startswith('stgit.alias.'):
             alias = name.replace('stgit.alias.', '', 1)
-            aliases[alias] = values[0]
+            command = values[0]
+            aliases.append((alias, command))
 
     put(
         '''\
@@ -150,7 +149,7 @@ def put(*args, **kwargs):
                 return 1
         end
     end
-end''' % ' '.join(aliases)
+end''' % ' '.join(alias for alias, _ in aliases)
     )
 
     put(
@@ -161,7 +160,7 @@ def put(*args, **kwargs):
     set --erase tokens[1 2]
     switch "$cmd"'''
     )
-    for alias, command in sorted(aliases.items()):
+    for alias, command in aliases:
         put('        case', alias)
         put('            set --prepend tokens', command)
     put(
@@ -172,12 +171,12 @@ def put(*args, **kwargs):
 '''
     )
 
-    put('### Aliases: %s' % ' '.join(aliases))
+    put('### Aliases: %s' % ' '.join(alias for alias, _ in aliases))
     put(
         "complete    -c stg -n '__fish_stg_is_alias' -x",
         "-a '(__fish_stg_complete_alias)'"
     )
-    for alias, command in sorted(aliases.items()):
+    for alias, command in aliases:
         put(
             "complete    -c stg -n '__fish_use_subcommand' -x",
             """-a %s -d 'Alias for "%s"'""" % (alias, command),
@@ -190,19 +189,19 @@ def put(*args, **kwargs):
         "complete -f -c stg -n '__fish_use_subcommand' -x",
         "-a help -d 'print the detailed command usage'",
     )
-    for cmd, (modname, _, _) in sorted(commands.items()):
+    for cmd, modname, _, _ in commands:
         mod = stgit.commands.get_command(modname)
         put(
             "complete -f -c stg -n '__fish_seen_subcommand_from help'",
             "-a %s -d '%s'" % (cmd, mod.help),
         )
-    for alias, command in sorted(aliases.items()):
+    for alias, command in aliases:
         put(
             "complete -f -c stg -n '__fish_seen_subcommand_from help'",
             """-a %s -d 'Alias for "%s"'""" % (alias, command),
         )
 
-    for cmd, (modname, _, _) in sorted(commands.items()):
+    for cmd, modname, _, _ in commands:
         mod = stgit.commands.get_command(modname)
         completions = []
         put('', '### %s' % cmd, sep='\n')

stgit/config.py

@@ -30,23 +30,23 @@
 along with this program; if not, see http://www.gnu.org/licenses/.
 """
 
-DEFAULTS = {
-    'stgit.smtpserver': ['localhost:25'],
-    'stgit.smtpdelay': ['5'],
-    'stgit.pullcmd': ['git pull'],
-    'stgit.fetchcmd': ['git fetch'],
-    'stgit.pull-policy': ['pull'],
-    'stgit.autoimerge': ['no'],
-    'stgit.keepoptimized': ['no'],
-    'stgit.refreshsubmodules': ['no'],
-    'stgit.shortnr': ['5'],
-    'stgit.pager': ['less'],
-    'stgit.alias.add': ['git add'],
-    'stgit.alias.rm': ['git rm'],
-    'stgit.alias.mv': ['git mv'],
-    'stgit.alias.resolved': ['git add'],
-    'stgit.alias.status': ['git status -s']
-}
+DEFAULTS = [
+    ('stgit.alias.add', ['git add']),
+    ('stgit.alias.mv', ['git mv']),
+    ('stgit.alias.resolved', ['git add']),
+    ('stgit.alias.rm', ['git rm']),
+    ('stgit.alias.status', ['git status -s']),
+    ('stgit.autoimerge', ['no']),
+    ('stgit.fetchcmd', ['git fetch']),
+    ('stgit.keepoptimized', ['no']),
+    ('stgit.pager', ['less']),
+    ('stgit.pull-policy', ['pull']),
+    ('stgit.pullcmd', ['git pull']),
+    ('stgit.refreshsubmodules', ['no']),
+    ('stgit.shortnr', ['5']),
+    ('stgit.smtpdelay', ['5']),
+    ('stgit.smtpserver', ['localhost:25']),
+]
 
 
 class GitConfigException(StgException):
@@ -61,7 +61,7 @@ def load(self):
         """Load the configuration in _cache unless it has been done already."""
         if self._cache is not None:
             return
-        self._cache = DEFAULTS.copy()
+        self._cache = dict(DEFAULTS)
         lines = Run('git', 'config', '--null', '--list'
                     ).discard_exitcode().output_lines('\0')
         for line in lines:

stgit/main.py

@@ -58,8 +58,9 @@ def is_cmd_alias(cmd):
 def append_alias_commands(cmd_list):
     for (name, command) in config.getstartswith('stgit.alias.'):
         name = utils.strip_prefix('stgit.alias.', name)
-        cmd_list[name] = (CommandAlias(name, command),
-                          'Alias commands', command)
+        cmd_list.append(
+            (name, CommandAlias(name, command), 'Alias commands', command)
+        )
 
 
 #
@@ -96,8 +97,7 @@ def __getitem__(self, key):
 
 cmd_list = stgit.commands.get_commands()
 append_alias_commands(cmd_list)
-commands = Commands((cmd, mod) for cmd, (mod, kind, help)
-                    in cmd_list.items())
+commands = Commands((cmd, mod) for cmd, mod, _, _ in cmd_list)
 
 
 def print_help():

stgit/utils.py

@@ -105,13 +105,6 @@ def strip_prefix(prefix, string):
     return string[len(prefix):]
 
 
-def strip_suffix(suffix, string):
-    """Return string, without the suffix. Blow up if string doesn't
-    end with suffix."""
-    assert string.endswith(suffix)
-    return string[:-len(suffix)]
-
-
 def create_dirs(directory):
     """Create the given directory, if the path doesn't already exist."""
     if directory and not os.path.isdir(directory):