diff --git a/project/config/settings/common.py b/project/config/settings/common.py index 7a3a745f..675ef47d 100644 --- a/project/config/settings/common.py +++ b/project/config/settings/common.py @@ -65,6 +65,7 @@ 'ndaparser', 'holviapp', 'velkoja', + 'slacksync', ) # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps @@ -270,6 +271,9 @@ HOLVI_APIKEY = env('HOLVI_APIKEY', default=None) HOLVI_BARCODE_IBAN = env('HOLVI_BARCODE_IBAN', default=None) HOLVI_NOTIFICATION_INTERVAL_DAYS = env('HOLVI_NOTIFICATION_INTERVAL_DAYS', default=7) +SLACK_APIKEY = env('SLACK_APIKEY', default=None) +SLACK_API_USERNAME = env('SLACK_API_USERNAME', default=None) +SLACK_INVITE_LINK = env('SLACK_INVITE_LINK', default=None) REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': [ diff --git a/project/creditor/management/commands/update_membershipfees.py b/project/creditor/management/commands/update_membershipfees.py index 10cf50ed..bc0db0af 100644 --- a/project/creditor/management/commands/update_membershipfees.py +++ b/project/creditor/management/commands/update_membershipfees.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import datetime -import dateutil.parser +import dateutil.parser from creditor.models import RecurringTransaction, TransactionTag from creditor.tests.fixtures.recurring import MembershipfeeFactory from django.core.management.base import BaseCommand, CommandError @@ -26,7 +26,7 @@ def handle(self, *args, **options): end=None, start__lt=cutoff_dt, amount=options['oldamount'] - ): + ): rt.end = end_dt rt.save() newrt = MembershipfeeFactory.create(amount=options['newamount'], start=cutoff_dt, end=None, owner=rt.owner) diff --git a/project/creditor/tests/fixtures/recurring.py b/project/creditor/tests/fixtures/recurring.py index a00ef66c..48728d53 100644 --- a/project/creditor/tests/fixtures/recurring.py +++ b/project/creditor/tests/fixtures/recurring.py @@ -9,11 +9,13 @@ from .tags import TransactionTagFactory + def get_tag(): if TransactionTag.objects.count(): return factory.fuzzy.FuzzyChoice(TransactionTag.objects.all()) return factory.SubFactory(TransactionTagFactory, label='Membership fee', tmatch='1') + class RecurringTransactionFactory(factory.django.DjangoModelFactory): class Meta: diff --git a/project/examples/handlers.py b/project/examples/handlers.py index a3252aa3..93bf2e94 100644 --- a/project/examples/handlers.py +++ b/project/examples/handlers.py @@ -166,9 +166,9 @@ def import_generic_transaction(self, at, lt): return lt def import_tmatch_transaction(self, at, lt): - if len(at.reference) < 2: # To avoid indexerrors + if len(at.reference) < 2: # To avoid indexerrors return None - if at.reference[0:2] == "RF": # ISO references, our lookup won't work with them, even worse: there will be exceptions + if at.reference[0:2] == "RF": # ISO references, our lookup won't work with them, even worse: there will be exceptions return None # In this example the last meaningful number (last number is checksum) of the reference is used to recognize the TransactionTag try: diff --git a/project/ndaparser/models.py b/project/ndaparser/models.py index 53cdbc37..f829a242 100644 --- a/project/ndaparser/models.py +++ b/project/ndaparser/models.py @@ -1,23 +1,20 @@ # -*- coding: utf-8 -*- import datetime -import slugify as unicodeslugify -from django.db import models, transaction +import slugify as unicodeslugify from django.conf import settings from django.contrib.auth import get_user_model +from django.db import models, transaction from django.utils.translation import ugettext_lazy as _ - from asylum.models import AsylumModel - def get_sentinel_user(): """Gets a "sentinel" user ("deleted") and for assigning as uploader""" return get_user_model().objects.get_or_create(username='deleted')[0] - def datestamped_and_normalized(instance, filename): """Normalized filename and places in datestamped path""" file_parts = filename.split('.') @@ -34,7 +31,6 @@ def datestamped_and_normalized(instance, filename): return datetime.datetime.now().strftime("ndaparser/%Y/%m/%d/{}").format(filename_normalized) - class UploadedTransaction(AsylumModel): """Track uploaded transaction files""" user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET(get_sentinel_user)) @@ -45,4 +41,4 @@ class UploadedTransaction(AsylumModel): class Meta: verbose_name = _('Uploaded transaction') verbose_name_plural = _('Uploaded transaction') - ordering = [ '-stamp' ] + ordering = ['-stamp'] diff --git a/project/ndaparser/views.py b/project/ndaparser/views.py index 665cce7e..5cec0762 100644 --- a/project/ndaparser/views.py +++ b/project/ndaparser/views.py @@ -7,8 +7,8 @@ from .forms import UploadForm from .importer import NDAImporter -from .parser import parseLine from .models import UploadedTransaction +from .parser import parseLine class NordeaUploadView(FormView): @@ -38,7 +38,7 @@ def form_valid(self, form): last_stamp = None with open(tmp.name) as fp: for line in fp: - nt = parseLine(line) + nt = parseLine(line) if not nt: continue if not last_stamp: @@ -47,9 +47,9 @@ def form_valid(self, form): last_stamp = nt.timestamp UploadedTransaction( - last_transaction = last_stamp, - file = self.request.FILES['ndafile'], - user = self.request.user + last_transaction=last_stamp, + file=self.request.FILES['ndafile'], + user=self.request.user ).save() # Done with the temp file, get rid of it diff --git a/project/requirements/base.txt b/project/requirements/base.txt index f88843f7..3727a5bc 100644 --- a/project/requirements/base.txt +++ b/project/requirements/base.txt @@ -41,3 +41,4 @@ django-markdown==0.8.4 django-settings-export==1.2.1 holviapi==0.3.20171118 python-dateutil==2.6.0 +slacker==0.9.50 diff --git a/project/slacksync/__init__.py b/project/slacksync/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/project/slacksync/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/project/slacksync/admin.py b/project/slacksync/admin.py new file mode 100644 index 00000000..34eec6ab --- /dev/null +++ b/project/slacksync/admin.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +from django.contrib import admin + +# Register your models here. diff --git a/project/slacksync/management/__init__.py b/project/slacksync/management/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/project/slacksync/management/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/project/slacksync/management/commands/__init__.py b/project/slacksync/management/commands/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/project/slacksync/management/commands/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/project/slacksync/management/commands/sync_slack_users.py b/project/slacksync/management/commands/sync_slack_users.py new file mode 100644 index 00000000..2450ba9a --- /dev/null +++ b/project/slacksync/management/commands/sync_slack_users.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from django.core.management.base import BaseCommand, CommandError +from slacksync.membersync import SlackMemberSync +from slacksync.utils import api_configured + + +class Command(BaseCommand): + help = 'Make sure all members are in Slack and optionally kick non-members' + + def add_arguments(self, parser): + parser.add_argument('--autodeactivate', action='store_true', help='Automatically deactivate users that are no longer members') + parser.add_argument('--noresend', action='store_true', help='Do not resend the invitation mail') + + pass + + def handle(self, *args, **options): + if not api_configured(): + raise CommandError("API not configured") + autoremove = False + if options['autodeactivate']: + autoremove = True + resend = True + if options['noresend']: + resend = False + sync = SlackMemberSync() + tbd = sync.sync_members(autoremove, resend) + if options['verbosity'] > 1: + for dm in tbd: + if autoremove: + print("User {uid} ({email}) was removed".format(uid=dm[0], email=dm[1])) + else: + print("User {uid} ({email}) should be removed".format(uid=dm[0], email=dm[1])) diff --git a/project/slacksync/membersync.py b/project/slacksync/membersync.py new file mode 100644 index 00000000..65a8dcd4 --- /dev/null +++ b/project/slacksync/membersync.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +import collections +import logging +import time + +import slacker +from django.conf import settings +from django.core.mail import send_mail +from members.models import Member +from requests.exceptions import RequestException +from requests.sessions import Session + +from .utils import api_configured, get_client + +logger = logging.getLogger() + + +class SlackMemberSync(object): + """Sync members and slack members""" + + def get_slack_users_simple(self, slack, exclude_api_user=True): + """Get just the properties we need from the slack members list""" + response = slack.users.list() + emails = [] + for member in response.body['members']: + if 'email' not in member['profile']: + # bot or similar + continue + if exclude_api_user and member['name'] == settings.SLACK_API_USERNAME: + continue + emails.append((member['id'], member['name'], member['profile']['email'])) + return emails + + def email_slack_link(self, member): + send_mail( + "Slack invite to {}".format(settings.ORGANIZATION_NAME), + "Click on the link to continue\n\n{}\n".format(settings.SLACK_INVITE_LINK), + settings.DEFAULT_FROM_EMAIL, + [member.email], + fail_silently=True + ) + + def sync_members(self, autodeactivate=False, resend=True): + """Sync members, NOTE: https://github.com/ErikKalkoken/slackApiDoc/blob/master/users.admin.setInactive.md says + deactivation via API works only on paid tiers""" + if not api_configured(): + raise RuntimeError("Slack API not configured") + with Session() as session: + slack = get_client(session=session) + slack_users = self.get_slack_users_simple(slack) + slack_emails = set([x[2] for x in slack_users]) + add_members = collections.deque(Member.objects.exclude(email__in=slack_emails)) + + while add_members: + member = add_members.popleft() + # If we have configured invite-link use it instead of API (which might hit the "too many invites" -error + if settings.SLACK_INVITE_LINK: + self.email_slack_link(member) + continue + try: + resp = slack.users.admin.invite(member.email, resend=resend) + if 'ok' not in resp.body or not resp.body['ok']: + self.logger.error("Could not invite {}, response: {}".format(member.email, resp.body)) + time.sleep(0.25) # rate-limit + except slacker.Error as e: + if str(e) == 'sent_recently': + continue + if str(e) == 'invalid_email': + logger.error("Slack says {} is invalid_email".format(member.email)) + continue + raise e + except RequestException as e: + if 'Retry-After' in e.response.headers: + wait_s = int(e.response.headers['Retry-After']) + logger.warning("Asked to wait {}s before retrying invite for {}".format(wait_s, member.email)) + time.sleep(wait_s * 1.5) + add_members.append(member) + continue + else: + logger.exception("Got exception when trying to invite {}".format(member.email)) + raise e + + member_emails = set(Member.objects.values_list('email', flat=True)) + remove_slack_emails = slack_emails - member_emails + remove_usernames = [] + if not remove_slack_emails: + return remove_usernames + + usernames_by_email = {x[2]: x[1] for x in slack_users} + remove_usernames = [(usernames_by_email[x], x) for x in remove_slack_emails] + + if not autodeactivate: + return remove_usernames + + userids_by_email = {x[2]: x[0] for x in slack_users} + remove_iter = collections.deque(remove_slack_emails) + while remove_iter: + email = remove_iter.popleft() + try: + resp = slack.users.admin.setInactive(userids_by_email[email]) + if 'ok' not in resp.body or not resp.body['ok']: + self.logger.error("Could not deactivate {}, response: {}".format(email, resp.body)) + time.sleep(0.25) # rate-limit + except RequestException as e: + if 'Retry-After' in e.response.headers: + wait_s = int(e.response.headers['Retry-After']) + logger.warning("Asked to wait {}s before retrying deactivation for {}".format(wait_s, email)) + time.sleep(wait_s * 1.5) + remove_iter.append(email) + continue + else: + logger.exception("Got exception when trying to deactivate {}".format(email)) + raise e + + return remove_usernames diff --git a/project/slacksync/migrations/__init__.py b/project/slacksync/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project/slacksync/models.py b/project/slacksync/models.py new file mode 100644 index 00000000..a978a7cd --- /dev/null +++ b/project/slacksync/models.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +from django.db import models + +# Create your models here. diff --git a/project/slacksync/tests.py b/project/slacksync/tests.py new file mode 100644 index 00000000..3c380430 --- /dev/null +++ b/project/slacksync/tests.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +from django.test import TestCase + +# Create your tests here. diff --git a/project/slacksync/utils.py b/project/slacksync/utils.py new file mode 100644 index 00000000..cc4c455a --- /dev/null +++ b/project/slacksync/utils.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +import logging + +from django.conf import settings +from slacker import Slacker + +logger = logging.getLogger() + + +def api_configured(): + """Check that Slack API settings are configured""" + return bool(settings.SLACK_APIKEY) and bool(settings.SLACK_API_USERNAME) + + +def get_client(**kwargs): + """Get a Slacker instance""" + if not api_configured(): + return False + return Slacker(settings.SLACK_APIKEY, **kwargs) + + +def quick_invite(email): + """Quickly invite single email""" + if not api_configured(): + return False + slack = get_client() + try: + resp = slack.users.admin.invite(email) + if 'ok' not in resp.body or not resp.body['ok']: + self.logger.error("Could not invite {}, response: {}".format(email, response.body)) + return False + except Exception as e: + logger.exception("Got exception when trying to invite {}".format(email)) + return False + return True diff --git a/project/slacksync/views.py b/project/slacksync/views.py new file mode 100644 index 00000000..d3ff201c --- /dev/null +++ b/project/slacksync/views.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +from django.shortcuts import render + +# Create your views here.