ParaTools Enhancements to Rift for S3-Based Annex, and more

.gitignore

@@ -1,3 +1,4 @@
+build
 
 # Compiled python modules.
 *.pyc

lib/rift/Config.py

@@ -86,6 +86,8 @@ class RiftDeprecatedConfWarning(FutureWarning):
 _DEFAULT_SYNC_METHOD = 'dnf'
 _DEFAULT_SYNC_INCLUDE = []
 _DEFAULT_SYNC_EXCLUDE = []
+_DEFAULT_S3_CREDENTIAL_FILE = '~/.rift/auth.json'
+
 
 class Config():
     """
@@ -111,9 +113,33 @@ class Config():
         'packages_dir': {
             'default':   _DEFAULT_PKG_DIR,
         },
+        's3_credential_file': {
+            'required': False,
+            'default':  _DEFAULT_S3_CREDENTIAL_FILE,
+        },
+        's3_auth_endpoint': {
+            'required': False,
+        },
+        'idp_auth_endpoint': {
+            'required': False
+        },
+        'idp_app_token': {
+            'required': True
+        },
+        'annex_restore_cache': {
+            'required': False,
+        },
         'annex': {
             'required': True,
         },
+        'annex_is_s3': {
+            'required': False,
+            'default': False,
+            'check': 'bool',
+        },
+        'annex_push': {
+            'required': False,
+        },
         'working_repo': {
         },
         'repos': {
@@ -547,6 +573,12 @@ def _key_value(self, syntax, key, value, check):
             "bool": bool,
         }
 
+        if check == 'bool':
+            if not isinstance(value, bool):
+                raise DeclError(
+                    f"Bad data type {value.__class__.__name__} for '{key}'"
+                )
+            return value
         if check == 'dict':
             if not isinstance(value, dict):
                 raise DeclError(

lib/rift/Controller.py

@@ -40,6 +40,7 @@
 from operator import attrgetter
 import random
 import time
+import datetime
 import textwrap
 # Since pylint can not found rpm.error, disable this check
 from rpm import error as RpmError # pylint: disable=no-name-in-module
@@ -48,7 +49,7 @@
 from rift import RiftError, __version__
 from rift.Annex import Annex, is_binary
 from rift.Config import Config, Staff, Modules
-from rift.Gerrit import Review
+from rift.auth import auth
 from rift.Mock import Mock
 from rift.Package import Package, Test
 from rift.Repository import LocalRepository, ProjectArchRepositories
@@ -216,6 +217,9 @@ def make_parser():
     subsubprs.add_argument('--dest', metavar='PATH', required=True,
                            help='destination path')
 
+    # Auth options
+    subprs = subparsers.add_parser('auth', help='Authenticate for access to Annex write access')
+
     # VM options
     subprs = subparsers.add_parser('vm', help='Manipulate VM process')
     subprs.add_argument('-a', '--arch', help='CPU architecture of the VM')
@@ -265,11 +269,9 @@ def make_parser():
     subprs.add_argument('--bump', dest='bump', action='store_true',
                         help='also bump the release number')
 
-    # Gerrit review
-    subprs = subparsers.add_parser('gerrit', add_help=False,
-                                   help='Make Gerrit automatic review')
-    subprs.add_argument('--change', help="Gerrit Change-Id", required=True)
-    subprs.add_argument('--patchset', help="Gerrit patchset ID", required=True)
+    # GitLab review
+    subprs = subparsers.add_parser('gitlab', add_help=False,
+                                   help='Check specfiles for GitLab')
     subprs.add_argument('patch', metavar='PATCH', type=argparse.FileType('r'))
 
     # sync
@@ -358,8 +360,8 @@ def action_annex(args, config, staff, modules):
                 message(f"{srcfile}: not an annex pointer, ignoring")
 
     elif args.annex_cmd == 'delete':
-        annex.delete(args.id)
-        message(f"{args.id} has been deleted")
+        if annex.delete(args.id):
+            message('%s has been deleted' % args.id)
 
     elif args.annex_cmd == 'get':
         annex.get(args.id, args.dest)
@@ -370,8 +372,26 @@ def action_annex(args, config, staff, modules):
         output_file = annex.backup(
             Package.list(config, staff, modules), args.output_file
         )
+
         message(f"Annex backup is available here: {output_file}")
 
+def action_auth(args, config):
+    """Action for 'auth' sub-commands."""
+    auth_obj = auth(config)
+
+    if auth_obj.authenticate():
+        msg = "succesfully authenticated"
+
+        t = auth_obj.get_expiration_timestr()
+        if t != "":
+            msg += "; token expires in {}".format(t)
+        else:
+            msg += "; token expiration time is unknown"
+
+        message(msg)
+    else:
+        message("error: authentication failed")
+
 def _vm_start(vm):
     if vm.running():
         message('VM is already running')
@@ -921,23 +941,18 @@ def action_validdiff(args, config):
 
     return rc
 
-def action_gerrit(args, config, staff, modules):
-    """Review a patchset for Gerrit (specfiles)"""
-
-    review = Review()
+def action_gitlab(args, config, staff, modules):
+    """Review a patchset for GitLab (specfiles)"""
 
     # Parse matching diff and specfiles in it
-    for patchedfile in parse_unidiff(args.patch):
-        filepath = patchedfile.path
-        names = filepath.split(os.path.sep)
+    for f in parse_unidiff(args.patch):
+        path = f.path
+        names = path.split(os.path.sep)
         if names[0] == config.get('packages_dir'):
             pkg = Package(names[1], config, staff, modules)
-            if filepath == pkg.specfile and not patchedfile.is_deleted_file:
-                Spec(pkg.specfile, config=config).analyze(review, pkg.dir)
-
-    # Push review
-    review.msg_header = 'rpmlint analysis'
-    review.push(config, args.change, args.patchset)
+            if os.path.abspath(path) == pkg.specfile and not f.is_deleted_file:
+                spec = Spec(pkg.specfile, config=config)
+                spec.check()
 
 def action_sync(args, config):
     """Action for 'sync' command."""
@@ -1043,6 +1058,11 @@ def action(config, args):
         action_annex(args, config, *staff_modules(config))
         return
 
+    # AUTH
+    if args.command == 'auth':
+        action_auth(args, config)
+        return
+
     # VM
     if args.command == 'vm':
         return action_vm(args, config)
@@ -1185,9 +1205,9 @@ def action(config, args):
              config=config).add_changelog_entry(author, args.comment,
                                                 bump=getattr(args, 'bump', False))
 
-    # GERRIT
-    elif args.command == 'gerrit':
-        return action_gerrit(args, config, *staff_modules(config))
+    # GITLAB
+    elif args.command == 'gitlab':
+        return action_gitlab(args, config, *staff_modules(config))
 
     # SYNC
     elif args.command == 'sync':

lib/rift/RPM.py

@@ -475,39 +475,13 @@ def check(self, pkg=None):
             if popen.returncode != 0:
                 raise RiftError(stderr or 'rpmlint reported errors')
 
-    def analyze(self, review, configdir=None):
-        """Run `rpmlint' for this specfile and fill provided `review'."""
-        cmd, env = self._check(configdir)
-        with Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, universal_newlines=True) as popen:
-            stdout, stderr = popen.communicate()
-            if popen.returncode not in (0, 64, 66):
-                raise RiftError(stderr or f"rpmlint returned {popen.returncode}")
-
-        for line in stdout.splitlines():
-            if line.startswith(self.filepath + ':'):
-                line = line[len(self.filepath + ':'):]
-                try:
-                    linenbr = None
-                    code, txt = line.split(':', 1)
-                    if code.isdigit():
-                        linenbr = int(code)
-                        code, txt = txt.split(':', 1)
-                    review.add_comment(self.filepath, linenbr,
-                                       code.strip(), txt.strip())
-                except (ValueError, KeyError):
-                    pass
-
-        if popen.returncode != 0:
-            review.invalidate()
-
     def supports_arch(self, arch):
         """
         Returns True is package spec file does not restrict ExclusiveArch or if
         the arch in argument is explicitely set in package ExclusiveArch.
         """
         return not self.exclusive_archs or arch in self.exclusive_archs
 
-
 class Variable():
 
     """