Merge pull request #2 from cevoaustralia/add-tests

Enabled TravisCI and fixed SNS issue

Author: stevemac007

.travis.yml

@@ -0,0 +1,26 @@
+language: python
+python:
+  - "2.6"
+  - "2.7"
+#  - "3.2"
+#  - "3.3"
+#  - "3.4"
+#  - "3.5"
+#  - "3.5-dev" # 3.5 development branch
+#  - "3.6"
+#  - "3.6-dev" # 3.6 development branch
+#  - "3.7-dev" # 3.7 development branch
+#  - "nightly" # currently points to 3.7-dev
+# command to install dependencies
+install: 
+  - "pip install -r requirements.txt"
+  - "pip install -r development.txt"
+env:
+  - PYTHONPATH=lambda
+# command to run tests
+script: nosetests --with-xunit  --with-coverage --cover-package=lambda --cover-erase
+
+notifications:
+  slack:
+    on_success: always
+    on_failure: always

README.md

@@ -1,6 +1,8 @@
 aws-backup-lambda
 =================
 
+[![Build Status](https://travis-ci.org/cevoaustralia/aws-backup-lambda.svg?branch=master)](https://travis-ci.org/cevoaustralia/aws-backup-lambda)
+
 A utility AWS lambda function to manage EBS and RDS snapshot backups.
 
  The Lambda function takes new backups when executed, and manages the deletion of the old ones when the upper limit is reached.

cloudformation.yaml

@@ -1,6 +1,28 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Transform: AWS::Serverless-2016-10-31
 Description: A simple backup process for EC2 and RDS datastores, tag your resources with 'MakeSnapshot' = 'True' (casesensitive) to have snapshots taken and managed
+Parameters:
+    SuccessSNSTopicOption:
+        Description: Supply an either '' (disabled), 'CreateSNS' (build new SNS for this stack), '<sns_arn>' (connect to existing Topic).
+        Default: ""
+        Type: String
+    ErrorSNSTopicOption:
+        Description: Supply an either '' (disabled), 'CreateSNS' (build new SNS for this stack), '<sns_arn>' (connect to existing Topic).
+        Default: ""
+        Type: String
+
+Conditions:
+    EnableSuccessSNSTopic: !Not [!Equals [!Ref SuccessSNSTopicOption, ""]] #
+    EnableErrorSNSTopic:   !Not [!Equals [!Ref ErrorSNSTopicOption, ""]]
+
+    CreateSuccessSNSTopic: !And                     # We only need to create the Success SNS when it is enabled and not supplied
+            - Condition: EnableSuccessSNSTopic
+            - !Equals [!Ref SuccessSNSTopicOption, "CreateSNS"]
+
+    CreateErrorSNSTopic:   !And                     # We only need to create the Error SNS when it is enabled and not supplied
+            - Condition: EnableErrorSNSTopic
+            - !Equals [!Ref ErrorSNSTopicOption,   "CreateSNS"]
+
 Resources:
   LambdaExecutionRole:
     Type: "AWS::IAM::Role"
@@ -60,7 +82,12 @@ Resources:
                     - "rds:CreateDBSnapshot"
                     - "rds:DeleteDBSnapshot"
                 Resource: "*"
-
+  SuccessSNSTopic:
+    Type: "AWS::SNS::Topic"
+    Condition : CreateSuccessSNSTopic
+  ErrorSNSTopic:
+    Type: "AWS::SNS::Topic"
+    Condition : CreateErrorSNSTopic
   BackupFunction:
     Type: AWS::Serverless::Function
     Properties:
@@ -78,6 +105,12 @@ Resources:
               Fn::Join:
               - ''
               - - '{"period_label": "day", "period_format": "%a%H-%M", "keep_count": 14,'
+                - '"arn": "'
+                - !If [EnableSuccessSNSTopic, !If [ CreateSuccessSNSTopic, Ref: SuccessSNSTopic, Ref: SuccessSNSTopicOption], '']
+                - '", '
+                - '"error_arn": "'
+                - !If [EnableErrorSNSTopic, !If [ CreateErrorSNSTopic, Ref: ErrorSNSTopic, Ref: ErrorSNSTopicOption], '']
+                - '", '
                 - '"ec2_region_name": "'
                 - Ref: AWS::Region
                 - '", '
@@ -93,6 +126,12 @@ Resources:
               Fn::Join:
               - ''
               - - '{"period_label": "week", "period_format": "week-%U", "keep_count": 12,'
+                - '"arn": "'
+                - !If [EnableSuccessSNSTopic, !If [ CreateSuccessSNSTopic, Ref: SuccessSNSTopic, Ref: SuccessSNSTopicOption], '']
+                - '", '
+                - '"error_arn": "'
+                - !If [EnableErrorSNSTopic, !If [ CreateErrorSNSTopic, Ref: ErrorSNSTopic, Ref: ErrorSNSTopicOption], '']
+                - '", '
                 - '"ec2_region_name": "'
                 - Ref: AWS::Region
                 - '", '

development.txt

@@ -0,0 +1,22 @@
+appdirs==1.4.0
+boto==2.45.0
+boto3==1.4.4
+botocore==1.5.7
+coverage==4.3.4
+docutils==0.13.1
+futures==3.0.5
+httpretty==0.8.10
+Jinja2==2.9.5
+jmespath==0.9.1
+MarkupSafe==0.23
+moto==0.4.31
+nose==1.3.7
+packaging==16.8
+pyparsing==2.1.10
+python-dateutil==2.6.0
+pytz==2016.10
+requests==2.13.0
+s3transfer==0.1.10
+six==1.10.0
+Werkzeug==0.11.15
+xmltodict==0.10.2

lambda/backuplambda.py

@@ -2,7 +2,7 @@
 
 import boto3
 from datetime import datetime
-import sys
+import sys, traceback
 import logging
 import json
 import pytz
@@ -43,6 +43,9 @@ def list_snapshots_for_resource(self, resource):
     def resolve_backupable_id(self, resource):
         pass
 
+    def resolve_snapshot_name(self, resource):
+        pass
+
     def resolve_snapshot_time(self, resource):
         return resource['StartTime']
 
@@ -86,9 +89,12 @@ def process_backup(self):
                     self.snapshot_resource(resource=backup_item, description=description, tags=tags_volume)
                     self.message += '    New Snapshot created with description: %s and tags: %s\n' % (description, str(tags_volume))
                     total_creates += 1
-                except Exception, e:
+                except Exception as e:
                     print ("Unexpected error:", sys.exc_info()[0])
                     print (e)
+                    exc_type, exc_value, exc_traceback = sys.exc_info()
+                    traceback.print_exception(exc_type, exc_value, exc_traceback,
+                                              limit=2, file=sys.stdout)
                     pass
 
                 snapshots = self.list_snapshots_for_resource(resource=backup_item)
@@ -104,11 +110,11 @@ def process_backup(self):
                     else:
                         print('  Skipping other backup schedule: ' + sndesc)
 
-                self.message += "\n    Current backups in rotation (keeping {})\n".format(self.keep_count)
+                self.message += "\n    Current backups in rotation (keeping {0})\n".format(self.keep_count)
                 self.message += "    ---------------------------\n"
 
                 for snap in deletelist:
-                    self.message += "    {} - {}\n".format(self.resolve_snapshot_name(snap),
+                    self.message += "    {0} - {1}\n".format(self.resolve_snapshot_name(snap),
                                                            self.resolve_snapshot_time(snap))
                 self.message += "    ---------------------------\n"
 
@@ -123,6 +129,9 @@ def process_backup(self):
             except Exception as ex:
                 print("Unexpected error:", sys.exc_info()[0])
                 print(ex)
+                exc_type, exc_value, exc_traceback = sys.exc_info()
+                traceback.print_exception(exc_type, exc_value, exc_traceback,
+                                          limit=2, file=sys.stdout)
                 logging.error('Error in processing volume with id: ' + backup_id)
                 self.errmsg += 'Error in processing volume with id: ' + backup_id
                 count_errors += 1
@@ -140,6 +149,13 @@ def process_backup(self):
         self.message += "\nTotal snapshots errors: " + str(count_errors)
         self.message += "\nTotal snapshots deleted: " + str(total_deletes) + "\n"
 
+        return {
+            "total_resources": count_total,
+            "total_creates": total_creates,
+            "total_errors": count_errors,
+            "total_deletes": total_deletes,
+        }
+
     def delete_snapshot(self, snapshot):
         pass
 
@@ -360,7 +376,7 @@ def build_arn_for_id(self, instance_id):
         region = self.conn.meta.region_name
         account_number = self.resolve_account_number()
 
-        return "arn:aws:rds:{}:{}:db:{}".format(region, account_number, instance_id)
+        return "arn:aws:rds:{0}:{1}:db:{2}".format(region, account_number, instance_id)
 
 
 def lambda_handler(event, context={}):
@@ -402,13 +418,6 @@ def lambda_handler(event, context={}):
 
     date_suffix = datetime.today().strftime(period_format)
 
-    sns_boto = None
-
-    # Connect to SNS
-    if sns_arn or error_sns_arn:
-        print('Connecting to SNS')
-        sns_boto = boto3.client('sns', region_name=ec2_region_name)
-
     result = event
     if ec2_region_name:
         backup_mgr = EC2BackupManager(ec2_region_name=ec2_region_name,
@@ -418,16 +427,24 @@ def lambda_handler(event, context={}):
                                       date_suffix=date_suffix,
                                       keep_count=keep_count)
 
-        backup_mgr.process_backup()
+        metrics = backup_mgr.process_backup()
 
+        result["metrics"] = metrics
         result["ec2_backup_result"] = backup_mgr.message
         print('\n' + backup_mgr.message + '\n')
 
+        sns_boto = None
+
+        # Connect to SNS
+        if sns_arn or error_sns_arn:
+            print('Connecting to SNS')
+            sns_boto = boto3.client('sns', region_name=ec2_region_name)
+
         if error_sns_arn and backup_mgr.errmsg:
             sns_boto.publish(error_sns_arn, 'Error in processing volumes: ' + backup_mgr.errmsg, 'Error with AWS Snapshot')
 
         if sns_arn:
-            sns_boto.publish(sns_arn, backup_mgr.message, 'Finished AWS EC2 snapshotting')
+            sns_boto.publish(TopicArn=sns_arn, Message=backup_mgr.message, Subject='Finished AWS EC2 snapshotting')
 
     if rds_region_name:
         backup_mgr = RDSBackupManager(rds_region_name=rds_region_name,
@@ -437,15 +454,23 @@ def lambda_handler(event, context={}):
                                       date_suffix=date_suffix,
                                       keep_count=keep_count)
 
-        backup_mgr.process_backup()
+        metrics = backup_mgr.process_backup()
 
+        result["metrics"] = metrics
         result["rds_backup_result"] = backup_mgr.message
         print('\n' + backup_mgr.message + '\n')
 
+        sns_boto = None
+
+        # Connect to SNS
+        if sns_arn or error_sns_arn:
+            print('Connecting to SNS')
+            sns_boto = boto3.client('sns', region_name=rds_region_name)
+
         if error_sns_arn and backup_mgr.errmsg:
             sns_boto.publish(error_sns_arn, 'Error in processing RDS: ' + backup_mgr.errmsg, 'Error with AWS Snapshot')
 
         if sns_arn:
-            sns_boto.publish(sns_arn, backup_mgr.message, 'Finished AWS RDS snapshotting')
+            sns_boto.publish(TopicArn=sns_arn, Message=backup_mgr.message, Subject='Finished AWS RDS snapshotting')
 
     return json.dumps(result, indent=2)