Add S3 support

Author: qa-cea

.gitignore

@@ -1,3 +1,4 @@
+build
 
 # Compiled python modules.
 *.pyc

lib/rift/Annex.py

@@ -77,6 +77,8 @@ def _construct_mapping(loader, node):
 _DEFAULT_REPO_CMD = 'createrepo_c'
 _DEFAULT_SHARED_FS_TYPE = '9p'
 _DEFAULT_VIRTIOFSD = '/usr/libexec/virtiofsd'
+_DEFAULT_S3_CREDENTIAL_FILE = '~/.rift/auth.json'
+
 
 class Config():
     """
@@ -102,9 +104,33 @@ class Config():
         'packages_dir': {
             'default':   _DEFAULT_PKG_DIR,
         },
+        's3_credential_file': {
+            'required': False,
+            'default':  _DEFAULT_S3_CREDENTIAL_FILE,
+        },
+        's3_auth_endpoint': {
+            'required': False,
+        },
+        'idp_auth_endpoint': {
+            'required': False
+        },
+        'idp_app_token': {
+            'required': True
+        },
+        'annex_restore_cache': {
+            'required': False,
+        },
         'annex': {
             'required': True,
         },
+        'annex_is_s3': {
+            'required': False,
+            'default': False,
+            'check': 'bool',
+        },
+        'annex_push': {
+            'required': False,
+        },
         'working_repo': {
         },
         'repos': {
@@ -179,11 +205,6 @@ class Config():
         'vm_build_post_script': {
             'default': _DEFAULT_VM_BUILD_POST_SCRIPT,
         },
-        'gerrit_realm': {},
-        'gerrit_server': {},
-        'gerrit_url': {},
-        'gerrit_username': {},
-        'gerrit_password': {},
         'rpm_macros': {
             'check':    'dict',
         },
@@ -417,8 +438,14 @@ def _key_value(self, syntax, key, value):
         """
         # Check type
         check = syntax.get('check', 'string')
-        assert check in ('string', 'dict', 'list', 'digit', 'enum')
+        assert check in ('string', 'dict', 'list', 'digit', 'enum', 'bool')
 
+        if check == 'bool':
+            if not isinstance(value, bool):
+                raise DeclError(
+                    f"Bad data type {value.__class__.__name__} for '{key}'"
+                )
+            return value
         if check == 'dict':
             if not isinstance(value, dict):
                 raise DeclError(

lib/rift/Config.py

@@ -40,6 +40,7 @@
 from operator import attrgetter
 import random
 import time
+import datetime
 import textwrap
 # Since pylint can not found rpm.error, disable this check
 from rpm import error as RpmError # pylint: disable=no-name-in-module
@@ -48,7 +49,7 @@
 from rift import RiftError, __version__
 from rift.Annex import Annex, is_binary
 from rift.Config import Config, Staff, Modules
-from rift.Gerrit import Review
+from rift.auth import auth
 from rift.Mock import Mock
 from rift.Package import Package, Test
 from rift.Repository import LocalRepository, ProjectArchRepositories
@@ -221,6 +222,9 @@ def make_parser():
     subsubprs.add_argument('--dest', metavar='PATH', required=True,
                            help='destination path')
 
+    # Auth options
+    subprs = subparsers.add_parser('auth', help='Authenticate for access to Annex write access')
+
     # VM options
     subprs = subparsers.add_parser('vm', help='Manipulate VM process')
     subprs.add_argument('-a', '--arch', help='CPU architecture of the VM')
@@ -270,11 +274,9 @@ def make_parser():
     subprs.add_argument('--bump', dest='bump', action='store_true',
                         help='also bump the release number')
 
-    # Gerrit review
-    subprs = subparsers.add_parser('gerrit', add_help=False,
-                                   help='Make Gerrit automatic review')
-    subprs.add_argument('--change', help="Gerrit Change-Id", required=True)
-    subprs.add_argument('--patchset', help="Gerrit patchset ID", required=True)
+    # GitLab review
+    subprs = subparsers.add_parser('gitlab', add_help=False,
+                                   help='Check specfiles for GitLab')
     subprs.add_argument('patch', metavar='PATCH', type=argparse.FileType('r'))
 
     # Parse options
@@ -357,8 +359,8 @@ def action_annex(args, config, staff, modules):
                 message('%s: not an annex pointer, ignoring' % srcfile)
 
     elif args.annex_cmd == 'delete':
-        annex.delete(args.id)
-        message('%s has been deleted' % args.id)
+        if annex.delete(args.id):
+            message('%s has been deleted' % args.id)
 
     elif args.annex_cmd == 'get':
         annex.get(args.id, args.dest)
@@ -369,8 +371,26 @@ def action_annex(args, config, staff, modules):
         output_file = annex.backup(
             Package.list(config, staff, modules), args.output_file
         )
+
         message(f"Annex backup is available here: {output_file}")
 
+def action_auth(args, config):
+    """Action for 'auth' sub-commands."""
+    auth_obj = auth(config)
+
+    if auth_obj.authenticate():
+        msg = "succesfully authenticated"
+
+        t = auth_obj.get_expiration_timestr()
+        if t != "":
+            msg += "; token expires in {}".format(t)
+        else:
+            msg += "; token expiration time is unknown"
+
+        message(msg)
+    else:
+        message("error: authentication failed")
+
 def _vm_start(vm):
     if vm.running():
         message('VM is already running')
@@ -874,23 +894,18 @@ def action_validdiff(args, config):
 
     return rc
 
-def action_gerrit(args, config, staff, modules):
-    """Review a patchset for Gerrit (specfiles)"""
-
-    review = Review()
+def action_gitlab(args, config, staff, modules):
+    """Review a patchset for GitLab (specfiles)"""
 
     # Parse matching diff and specfiles in it
-    for patchedfile in parse_unidiff(args.patch):
-        filepath = patchedfile.path
-        names = filepath.split(os.path.sep)
+    for f in parse_unidiff(args.patch):
+        path = f.path
+        names = path.split(os.path.sep)
         if names[0] == config.get('packages_dir'):
             pkg = Package(names[1], config, staff, modules)
-            if filepath == pkg.specfile and not patchedfile.is_deleted_file:
-                Spec(pkg.specfile, config=config).analyze(review, pkg.dir)
-
-    # Push review
-    review.msg_header = 'rpmlint analysis'
-    review.push(config, args.change, args.patchset)
+            if os.path.abspath(path) == pkg.specfile and not f.is_deleted_file:
+                spec = Spec(pkg.specfile, config=config)
+                spec.check()
 
 def get_packages_from_patch(patch, config, modules, staff):
     """
@@ -981,6 +996,11 @@ def action(config, args):
         action_annex(args, config, *staff_modules(config))
         return
 
+    # AUTH
+    if args.command == 'auth':
+        action_auth(args, config)
+        return
+
     # VM
     if args.command == 'vm':
         return action_vm(args, config)
@@ -1124,9 +1144,9 @@ def action(config, args):
              config=config).add_changelog_entry(author, args.comment,
                                                 bump=getattr(args, 'bump', False))
 
-    # GERRIT
-    elif args.command == 'gerrit':
-        return action_gerrit(args, config, *staff_modules(config))
+    # GITLAB
+    elif args.command == 'gitlab':
+        return action_gitlab(args, config, *staff_modules(config))
 
     return 0
 
@@ -1167,6 +1187,18 @@ def _validate_patched_file(patched_file, config, modules, staff):
     if patched_file.binary:
         raise RiftError("Binary file detected: %s" % filepath)
 
+    if filepath == '.gitlab-ci.yml':
+        logging.debug('Ignoring gitlab ci file: %s', filepath)
+        return False
+
+    if filepath == 'CODEOWNERS':
+        logging.debug('Ignoring gitlab ci file: %s', filepath)
+        return False
+
+    if names[0] == "gitlab-ci":
+        logging.debug("Ignoring gitlab ci file: %s" % filepath)
+        return False
+
     if names[0] != config.get('packages_dir'):
         raise RiftError("Unknown file pattern: %s" % filepath)