Merge pull request #37 from IT-CEREBRUM/CRB-3996-greg-import-setting

CRB-3996: GREG_IMPORT setting

Author: fredrikhl

Cerebrum/modules/greg/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2021 University of Oslo, Norway
+# Copyright 2021-2023 University of Oslo, Norway
 #
 # This file is part of Cerebrum.
 #
@@ -43,7 +43,11 @@
 Configuration
 -------------
 
-cereconf.CLASS_CONSTANTS
+``cereconf.CLASS_CONSTANTS``
     Must include ``Cerebrum.modules.greg.constants/GregConstants``, to provide
     Greg-related constants.
+
+``cereconf.GREG_IMPORT``
+    Can be used to override the default
+    ``Cerebrum.modules.greg.importer/GregImporter`` class in scripts.
 """

Cerebrum/modules/greg/importer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2021 University of Oslo, Norway
+# Copyright 2021-2023 University of Oslo, Norway
 #
 # This file is part of Cerebrum.
 #
@@ -17,7 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with Cerebrum; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-""" Greg person import/update.  """
+"""
+Greg person import/update.
+"""
 from __future__ import (
     absolute_import,
     division,
@@ -27,6 +29,8 @@
 import logging
 import datetime
 
+import cereconf
+
 from Cerebrum.Utils import Factory
 from Cerebrum.modules.import_utils.matcher import (
     OuMatcher,
@@ -39,14 +43,30 @@
     PersonNameSync,
 )
 from Cerebrum.utils import date_compat
+from Cerebrum.utils.module import resolve
 
-from .consent import sync_greg_consent
 from .datasource import GregDatasource
 from .mapper import GregMapper
 
 logger = logging.getLogger(__name__)
 
 
+def get_import_class(cereconf=cereconf):
+    """
+    Get preferred import class from config module/object *cereconf*.
+
+    TODO: Or should we re-factor the greg client config into a full greg
+    config, with import class and everything?
+    """
+    import_spec = getattr(cereconf, 'GREG_IMPORT', None)
+    if import_spec:
+        cls = resolve(import_spec)
+    else:
+        cls = GregImporter
+    logger.info("greg import class=%s", repr(cls))
+    return cls
+
+
 class GregImporter(object):
 
     REQUIRED_PERSON_ID = (
@@ -58,9 +78,9 @@ class GregImporter(object):
         'GREG_PID',
     )
 
-    CONSENT_GROUPS = {
-        'greg-publish': sync_greg_consent,
-    }
+    # Map consent name to
+    # `Cerebrum.modules.import_utils.group.GroupMembershipSetter`
+    CONSENT_GROUPS = {}
 
     mapper = GregMapper()
 

Cerebrum/modules/greg/tasks.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2021 University of Oslo, Norway
+# Copyright 2021-2023 University of Oslo, Norway
 #
 # This file is part of Cerebrum.
 #
@@ -19,7 +19,6 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
 """
 Tasks related to the Greg guest import.
-
 """
 import logging
 
@@ -33,12 +32,32 @@
 
 
 class GregImportTasks(queue_handler.QueueHandler):
-    """ This object defines the 'greg-guest' tasks queue. """
+    """ This object defines the 'greg-person' task queues. """
 
     queue = 'greg-person'
     manual_sub = 'manual'
     max_attempts = 20
 
+    def __init__(self, client, import_class):
+        self._client = client
+        self._import_class = import_class
+
+    def _callback(self, db, task):
+        greg_id = task.key
+        logger.info('Updating greg_id=%s', greg_id)
+        importer = self._import_class(db, client=self._client)
+        importer.handle_reference(greg_id)
+        logger.info('Updated greg_id=%s', greg_id)
+
+        # TODO: Should we have the importer return potential new tasks?  If so,
+        # we could rely on the default *handle_task* implementation for
+        # re-queueing.
+        #
+        # *Or* should the import itself add potential tasks to the queue?  It
+        # kind of depends on whether we want the option to run the importer
+        # *without* adding new tasks to the queue...
+        return []
+
     @classmethod
     def create_manual_task(cls, reference, sub=manual_sub, nbf=None):
         """ Create a manual task. """

Cerebrum/modules/greg/consent.py Cerebrum/modules/import_utils/groups.pyCerebrum/modules/greg/consent.py renamed to Cerebrum/modules/import_utils/groups.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2021 University of Oslo, Norway
+# Copyright 2021-2023 University of Oslo, Norway
 #
 # This file is part of Cerebrum.
 #
@@ -18,24 +18,13 @@
 # along with Cerebrum; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
 """
-Greg consent functionality.
+Sync group membership for a given entity.
 """
 import logging
 
-from Cerebrum.group.template import GroupTemplate
-
 logger = logging.getLogger(__name__)
 
 
-GREG_CONSENT_GROUP = GroupTemplate(
-    group_name='greg-aktivt-samtykke',
-    group_description='Guests who consents to electronic publication',
-    group_type='internal-group',
-    group_visibility='A',
-)
-
-
-# TODO: Should this be a 'generic' import_utils class?
 class GroupMembershipSetter(object):
     """
     Set membership for a single entity in a given group.
@@ -60,9 +49,8 @@ def __init__(self, get_group):
             connection/transaction to use, and should return the
             Cerebrum.Group.Group object to update.
 
-            Would typically be a
-            py:class:`Cerebrum.group.template.GroupTemplate` or similar
-            callable object.
+            Would typically be a :class:`Cerebrum.group.template.GroupTemplate`
+            or similar callable object.
         """
         self.get_group = get_group
 
@@ -73,17 +61,28 @@ def __repr__(self):
         )
 
     def __call__(self, db, entity_id, set_member):
+        """
+        Ensure entity_id is or isn't a member of this group.
+
+        :type db: Cerebrum.database.Database
+        :param int entity_id: member id to sync
+        :param bool set_member: if entity_id should be a member
+
+        :returns bool: True if membership was changed
+        """
         group = self.get_group(db)
         is_member = group.has_member(entity_id)
 
         if set_member and not is_member:
             logger.info('adding entity_id=%d to group %s (%d)',
                         entity_id, group.group_name, group.entity_id)
             group.add_member(entity_id)
-        elif not set_member and is_member:
+            return True
+
+        if not set_member and is_member:
             logger.info('removing entity_id=%d from group %s (%d)',
                         entity_id, group.group_name, group.entity_id)
             group.remove_member(entity_id)
+            return True
 
-
-sync_greg_consent = GroupMembershipSetter(GREG_CONSENT_GROUP)
+        return False

Cerebrum/modules/no/uio/greg_import.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2023 University of Oslo, Norway
+#
+# This file is part of Cerebrum.
+#
+# Cerebrum is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Cerebrum is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Cerebrum; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+"""
+UiO-specific Greg import logic.
+"""
+from Cerebrum.group.template import GroupTemplate
+from Cerebrum.modules.greg import mapper
+from Cerebrum.modules.greg import importer
+from Cerebrum.modules.import_utils.groups import GroupMembershipSetter
+
+
+GREG_CONSENT_GROUP = GroupTemplate(
+    group_name='greg-aktivt-samtykke',
+    group_description='Guests who consents to electronic publication',
+    group_type='internal-group',
+    group_visibility='A',
+)
+
+sync_greg_consent = GroupMembershipSetter(GREG_CONSENT_GROUP)
+
+
+class UioGregMapper(mapper.GregMapper):
+    pass
+
+
+class UioGregImporter(importer.GregImporter):
+
+    CONSENT_GROUPS = {
+        'greg-publish': sync_greg_consent,
+    }
+
+    mapper = UioGregMapper()

Cerebrum/modules/tasks/queue_handler.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2021 University of Oslo, Norway
+# Copyright 2021-2023 University of Oslo, Norway
 #
 # This file is part of Cerebrum.
 #
@@ -29,7 +29,10 @@
 logger = logging.getLogger(__name__)
 
 
-delay_on_error = backoff.Backoff(
+# Default backoff for errors/retries in QueueHandler.  This backoff yields
+# time deltas 03:45, 07:30, 15:00, 30:00, 1:00:00, ... - before truncating at
+# 12:00:00 after 10 attempts.  This should be a good backoff for most tasks.
+default_retry_delay = backoff.Backoff(
     backoff.Exponential(2),
     backoff.Factor(datetime.timedelta(hours=1) / 16),
     backoff.Truncate(datetime.timedelta(hours=12)),
@@ -60,16 +63,19 @@ class QueueHandler(object):
     # when to give up on a task
     max_attempts = 20
 
+    # next delay (timedelta) after *n* failed attempts
+    get_retry_delay = default_retry_delay
+
     def __init__(self, callback):
         self._callback = callback
 
     def get_retry_task(self, task, error):
         """ Create a retry task from a failed task. """
         retry = task_models.copy_task(task)
         retry.queue = self.queue
-        retry.sub = self.retry_sub or ""
+        retry.sub = self.retry_sub or task.sub
         retry.attempts = task.attempts + 1
-        retry.nbf = now() + delay_on_error(task.attempts + 1)
+        retry.nbf = now() + self.get_retry_delay(task.attempts + 1)
         retry.reason = 'retry: failed_at={} error={}'.format(now(), error)
         return retry
 

contrib/greg/greg-manual-import.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright 2021 University of Oslo, Norway
+# Copyright 2021-2023 University of Oslo, Norway
 #
 # This file is part of Cerebrum.
 #
@@ -31,15 +31,14 @@
     unicode_literals,
 )
 import argparse
-import functools
 import logging
 
 import Cerebrum.logutils
 import Cerebrum.logutils.options
 from Cerebrum.Utils import Factory
 from Cerebrum.database.ctx import db_context
 from Cerebrum.modules.greg.client import get_client
-from Cerebrum.modules.greg.importer import GregImporter
+from Cerebrum.modules.greg.importer import get_import_class
 from Cerebrum.utils.argutils import add_commit_args
 
 logger = logging.getLogger(__name__)
@@ -81,13 +80,11 @@ def main(inargs=None):
     logger.info("start %s", parser.prog)
     logger.debug("args: %r", args)
 
-    # we don't really need the full TaskImportConfig here, but it's easier to
-    # re-use the existing config.
     client = get_client(args.config)
-    get_import = functools.partial(GregImporter, client=client)
+    import_class = get_import_class()
 
     with db_context(get_db(), not args.commit) as db:
-        greg_import = get_import(db)
+        greg_import = import_class(db, client=client)
 
         logger.info('handle reference=%r', args.reference)
         greg_import.handle_reference(args.reference)