email notification when extraction is done

Author: bingzhang

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - files.upload_preview and collections.upload_preview does not require section_id for metadata [CATS-935](https://opensource.ncsa.illinois.edu/jira/browse/CATS-935)
 
 ### Added
+- Email notification when extraction is done [CATSPYC-17] (https://opensource.ncsa.illinois.edu/jira/browse/CATSPYC-17) 
 - Docker compose file for starting up the Clowder stack [BD-2226](https://opensource.ncsa.illinois.edu/jira/browse/BD-2226)
 
 ## 2.1.1 - 2018-07-12
@@ -40,4 +41,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Now has onbuild version of pyclowder
 - release.sh will now tag images (e.g. this will be tagged 2.0.3 2.0 and 2)
 - RABBITMQ_URI is now set to amqp://guest:guest@rabbitmq/%2F to allow easy deployment
-  using docker-compose of clowder
+  using docker-compose of clowder

Dockerfile

@@ -5,6 +5,8 @@ ENV RABBITMQ_URI="amqp://guest:guest@rabbitmq:5672/%2F" \
     RABBITMQ_EXCHANGE="clowder" \
     RABBITMQ_QUEUE="" \
     REGISTRATION_ENDPOINTS="" \
+    EMAIL_SERVER="" \
+    EMAIL_SENDER="extractor" \
     MAIN_SCRIPT=""
 
 # install python
@@ -23,6 +25,7 @@ RUN pip install --upgrade /tmp/pyclowder \
 
 # folder for pyclowder code
 WORKDIR /home/clowder
+COPY notifications.json /home/clowder
 
 # command to run when starting container
 CMD python "./${MAIN_SCRIPT}"

notifications.json

@@ -0,0 +1,11 @@
+{
+    "extractor_name": "ExtractorName",
+    "sender": "[email protected]",
+    "notifications": {
+        "email": {
+            "from": "${sender}",
+            "subject": "${extractor_name} is done",
+            "body": "This is email notification on extraction completion."
+        }
+    }
+}

pyclowder/connectors.py

@@ -48,6 +48,11 @@
 import pyclowder.files
 import pyclowder.utils
 
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from string import Template
+
 
 class Connector(object):
     """ Class that will listen for messages.
@@ -70,6 +75,49 @@ def __init__(self, extractor_name, extractor_info, check_message=None, process_m
         else:
             self.mounted_paths = mounted_paths
 
+        filename = 'notifications.json'
+        self.smtp_server = None
+        if os.path.isfile(filename):
+            try:
+                with open(filename) as notifications_file:
+                    notifications_content = notifications_file.read()
+                    notifications_template = Template(notifications_content)
+                    notifications_json = json.loads(notifications_content)
+                    notifications_json['extractor_name'] = extractor_name
+                    notifications = notifications_template.safe_substitute(notifications_json)
+
+                    notifications_interpolate = json.loads(notifications)
+                    self.smtp_server = os.getenv('EMAIL_SERVER', None)
+                    self.emailmsg = MIMEMultipart('alternative')
+
+                    self.emailmsg['From'] = os.getenv('EMAIL_SENDER', notifications_json.get('sender'))
+                    self.emailmsg['Subject'] = notifications_interpolate.get('notifications').get('email').get(
+                        'subject')
+                    self.emailmsg['Body'] = notifications_interpolate.get('notifications').get('email').get('body')
+            except Exception:  # pylint: disable=broad-except
+                print("Error loading notifications.json")
+
+    def email(self, emaillist, clowderurl):
+        """ Send extraction completion as the email notification """
+        logger = logging.getLogger(__name__)
+        if emaillist and self.smtp_server:
+            server = smtplib.SMTP(self.smtp_server)
+            msg = MIMEMultipart('alternative')
+            msg['Subject'] = self.emailmsg['Subject']
+            msg['From'] = self.emailmsg['From']
+
+            content = "%s \n%s" % (self.emailmsg['Body'], clowderurl)
+            content = MIMEText(content.encode('utf-8'), _charset='utf-8')
+            msg.attach(content)
+
+            try:
+                logger.debug("send email notification to %s, %s " % (emaillist, msg.as_string()))
+                server.sendmail(msg['From'], emaillist, msg.as_string())
+            except:
+                logger.warning("failed to send email notification to %s" % emaillist)
+                pass
+            server.quit()
+
     def listen(self):
         """Listen for incoming messages.
 
@@ -309,7 +357,10 @@ def _process_message(self, body):
         """
 
         logger = logging.getLogger(__name__)
-
+        emailaddrlist = None
+        if body.get('notifies'):
+            emailaddrlist = body.get('notifies')
+            logger.debug(emailaddrlist)
         host = body.get('host', '')
         if host == '':
             return
@@ -356,6 +407,10 @@ def _process_message(self, body):
                                 resource['local_paths'] = [file_path]
 
                             self.process_message(self, host, secret_key, resource, body)
+
+                            clowderurl = "%sfiles/%s" % (host, body.get('id', ''))
+                            # notificatino of extraction job is done by email.
+                            self.email(emailaddrlist, clowderurl)
                         finally:
                             if file_path is not None and not found_local:
                                 try:
@@ -372,6 +427,9 @@ def _process_message(self, body):
                             resource['local_paths'] = file_paths
 
                             self.process_message(self, host, secret_key, resource, body)
+                            clowderurl = "%sdatasets/%s" % (host, body.get('datasetId', ''))
+                            # notificatino of extraction job is done by email.
+                            self.email(emailaddrlist, clowderurl)
                         finally:
                             for tmp_f in tmp_files:
                                 try: