Merge branch 'develop' into extend-readme

Author: glennmatthews

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v2.0.3
+
+### Added
+
+- #91 - `Provider` now adds `_include_filter` and `_exclude_filter` attributes (using regex) to filter in and out notifications that are relevant to be parsed vs other that are not, avoiding false positives.
+
+## v2.0.2 - 2021-09-28
+
+### Fixed
+
+- #86 - Fix `CombinedProcessor` carries over data from previous parsing
+
+### Added
+
+- #84 - New parser added for text. Added new provider `AquaComms` using `Text` and `EmailSubjectParser`
+
 ## v2.0.1 - 2021-09-16
 
 ### Fixed

README.md

@@ -59,6 +59,7 @@ By default, there is a `GenericProvider` that support a `SimpleProcessor` using
 
 #### Supported providers based on other parsers
 
+- AWS
 - AquaComms
 - Cogent
 - Colt
@@ -193,7 +194,7 @@ There is also a `cli` entrypoint `circuit-maintenance-parser` which offers easy
 circuit-maintenance-parser --data-file "/tmp/___ZAYO TTN-00000000 Planned MAINTENANCE NOTIFICATION___.eml" --data-type email --provider-type zayo
 Circuit Maintenance Notification #0
 {
-  "account": "my_account",
+  "account": "some account",
   "circuits": [
     {
       "circuit_id": "/OGYX/000000/ /ZYO /",
@@ -297,6 +298,7 @@ The project is following Network to Code software development guidelines and is
 1. Define the `Parsers`(inheriting from some of the generic `Parsers` or a new one) that will extract the data from the notification, that could contain itself multiple `DataParts`. The `data_type` of the `Parser` and the `DataPart` have to match. The custom `Parsers` will be placed in the `parsers` folder.
 2. Update the `unit/test_parsers.py` with the new parsers, providing some data to test and validate the extracted data.
 3. Define a new `Provider` inheriting from the `GenericProvider`, defining the `Processors` and the respective `Parsers` to be used. Maybe you can reuse some of the generic `Processors` or maybe you will need to create a custom one. If this is the case, place it in the `processors` folder.
+   - The `Provider` also supports the definition of a `_include_filter` and a `_exclude_filter` to limit the notifications that are actually processed, avoiding false positive errors for notification that are not relevant.
 4. Update the `unit/test_e2e.py` with the new provider, providing some data to test and validate the final `Maintenances` created.
 5. **Expose the new `Provider` class** updating the map `SUPPORTED_PROVIDERS` in `circuit_maintenance_parser/__init__.py` to officially expose the `Provider`.
 

circuit_maintenance_parser/__init__.py

@@ -7,6 +7,7 @@
 from .provider import (
     GenericProvider,
     AquaComms,
+    AWS,
     Cogent,
     Colt,
     EUNetworks,
@@ -29,6 +30,7 @@
 SUPPORTED_PROVIDERS = (
     GenericProvider,
     AquaComms,
+    AWS,
     Cogent,
     Colt,
     EUNetworks,

circuit_maintenance_parser/constants.py

@@ -0,0 +1,4 @@
+"""Constants used in the library."""
+
+EMAIL_HEADER_SUBJECT = "email-header-subject"
+EMAIL_HEADER_DATE = "email-header-date"

circuit_maintenance_parser/data.py

@@ -4,6 +4,8 @@
 
 import email
 from pydantic import BaseModel, Extra
+from circuit_maintenance_parser.constants import EMAIL_HEADER_SUBJECT, EMAIL_HEADER_DATE
+
 
 logger = logging.getLogger(__name__)
 
@@ -73,9 +75,8 @@ def init_from_emailmessage(cls: Type["NotificationData"], email_message) -> Opti
             cls.walk_email(email_message, data_parts)
 
             # Adding extra headers that are interesting to be parsed
-            data_parts.add(DataPart("email-header-subject", email_message["Subject"].encode()))
-            # TODO: Date could be used to extend the "Stamp" time of a notification when not available, but we need a parser
-            data_parts.add(DataPart("email-header-date", email_message["Date"].encode()))
+            data_parts.add(DataPart(EMAIL_HEADER_SUBJECT, email_message["Subject"].encode()))
+            data_parts.add(DataPart(EMAIL_HEADER_DATE, email_message["Date"].encode()))
             return cls(data_parts=list(data_parts))
         except Exception:  # pylint: disable=broad-except
             logger.exception("Error found initializing data from email message: %s", email_message)

circuit_maintenance_parser/parser.py

@@ -15,6 +15,7 @@
 
 from circuit_maintenance_parser.errors import ParserError
 from circuit_maintenance_parser.output import Status, Impact, CircuitImpact
+from circuit_maintenance_parser.constants import EMAIL_HEADER_SUBJECT, EMAIL_HEADER_DATE
 
 # pylint: disable=no-member
 
@@ -177,7 +178,7 @@ def clean_line(line):
 class EmailDateParser(Parser):
     """Parser for Email Date."""
 
-    _data_types = ["email-header-date"]
+    _data_types = [EMAIL_HEADER_DATE]
 
     def parser_hook(self, raw: bytes):
         """Execute parsing."""
@@ -190,7 +191,7 @@ def parser_hook(self, raw: bytes):
 class EmailSubjectParser(Parser):
     """Parse data from subject or email."""
 
-    _data_types = ["email-header-subject"]
+    _data_types = [EMAIL_HEADER_SUBJECT]
 
     def parser_hook(self, raw: bytes):
         """Execute parsing."""
@@ -226,3 +227,26 @@ def parser_hook(self, raw: bytes):
     def parse_csv(raw: bytes) -> List[Dict]:
         """Custom CSV parsing."""
         raise NotImplementedError
+
+
+class Text(Parser):
+    """Text parser."""
+
+    _data_types = ["text/plain"]
+
+    def parser_hook(self, raw: bytes):
+        """Execute parsing."""
+        result = []
+        text = self.get_text_hook(raw)
+        for data in self.parse_text(text):
+            result.append(data)
+        return result
+
+    @staticmethod
+    def get_text_hook(raw: bytes) -> str:
+        """Can be overwritten by subclasses."""
+        return raw.decode()
+
+    def parse_text(self, text) -> List[Dict]:
+        """Custom text parsing."""
+        raise NotImplementedError

circuit_maintenance_parser/parsers/aws.py

@@ -0,0 +1,90 @@
+"""AquaComms parser."""
+import hashlib
+import logging
+import quopri
+import re
+
+import bs4  # type: ignore
+
+from dateutil import parser
+
+from circuit_maintenance_parser.parser import CircuitImpact, EmailSubjectParser, Impact, Status, Text
+
+# pylint: disable=too-many-nested-blocks, too-many-branches
+
+logger = logging.getLogger(__name__)
+
+
+class SubjectParserAWS1(EmailSubjectParser):
+    """Subject parser for AWS notifications."""
+
+    def parse_subject(self, subject):
+        """Parse subject.
+
+        Example: AWS Direct Connect Planned Maintenance Notification [AWS Account: 00000001]
+        """
+        data = {}
+        search = re.search(r"\[AWS Account ?I?D?: ([0-9]+)\]", subject)
+        if search:
+            data["account"] = search.group(1)
+        return [data]
+
+
+class TextParserAWS1(Text):
+    """Parse text body of email."""
+
+    @staticmethod
+    def get_text_hook(raw):
+        """Modify soup before entering `parse_text`."""
+        soup = bs4.BeautifulSoup(quopri.decodestring(raw), features="lxml")
+        return soup.text
+
+    def parse_text(self, text):
+        """Parse text.
+
+        Example:
+            Hello,
+
+            Planned maintenance has been scheduled on an AWS Direct Connect router in A=
+            Block, New York, NY from Thu, 20 May 2021 08:00:00 GMT to Thu, 20 Ma=
+            y 2021 14:00:00 GMT for 6 hours. During this maintenance window, your AWS D=
+            irect Connect services listed below may become unavailable.
+
+            aaaaa-00000001
+            aaaaa-00000002
+            aaaaa-00000003
+            aaaaa-00000004
+            aaaaa-00000005
+            aaaaa-00000006
+
+            This maintenance is scheduled to avoid disrupting redundant connections at =
+            the same time.
+        """
+        data = {"circuits": []}
+        impact = Impact.OUTAGE
+        maintenace_id = ""
+        status = Status.CONFIRMED
+        for line in text.splitlines():
+            if "planned maintenance" in line.lower():
+                data["summary"] = line
+                search = re.search(
+                    r"([A-Z][a-z]{2}, [0-9]{1,2} [A-Z][a-z]{2,9} [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{2,3}) to ([A-Z][a-z]{2}, [0-9]{1,2} [A-Z][a-z]{2,9} [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{2,3})",
+                    line,
+                )
+                if search:
+                    data["start"] = self.dt2ts(parser.parse(search.group(1)))
+                    data["end"] = self.dt2ts(parser.parse(search.group(2)))
+                    maintenace_id += str(data["start"])
+                    maintenace_id += str(data["end"])
+                if "may become unavailable" in line.lower():
+                    impact = Impact.OUTAGE
+                elif "has been cancelled" in line.lower():
+                    status = Status.CANCELLED
+            elif re.match(r"[a-z]{5}-[a-z0-9]{8}", line):
+                maintenace_id += line
+                data["circuits"].append(CircuitImpact(circuit_id=line, impact=impact))
+        # No maintenance ID found in emails, so a hash value is being generated using the start,
+        #  end and IDs of all circuits in the notification.
+        data["maintenance_id"] = hashlib.md5(maintenace_id.encode("utf-8")).hexdigest()  # nosec
+        data["status"] = status
+        return [data]

circuit_maintenance_parser/parsers/lumen.py

@@ -6,7 +6,7 @@
 import bs4  # type: ignore
 from bs4.element import ResultSet  # type: ignore
 
-from circuit_maintenance_parser.parser import Html, Impact, CircuitImpact, Status
+from circuit_maintenance_parser.parser import CircuitImpact, Html, Impact, Status
 
 # pylint: disable=too-many-nested-blocks, too-many-branches
 
@@ -88,8 +88,16 @@ def parse_tables(self, tables: ResultSet, data: Dict):
                     if "account" not in data:
                         data["account"] = cells[idx].string
                     if num_columns == 10:
-                        if cells[idx + 9].string == "Completed":
+                        status_string = cells[idx + 9].string
+                        if status_string == "Completed":
                             data["status"] = Status("COMPLETED")
+                        elif status_string == "Postponed":
+                            data["status"] = Status("RE-SCHEDULED")
+                        elif status_string == "Not Completed":
+                            data["status"] = Status("CANCELLED")
+                    elif "status" not in data:
+                        # Update to an existing ticket may not include an update to the status - make a guess
+                        data["status"] = "CONFIRMED"
 
                     data_circuit = {}
                     data_circuit["circuit_id"] = cells[idx + 1].string

circuit_maintenance_parser/processor.py

@@ -115,6 +115,11 @@ class CombinedProcessor(GenericProcessor):
     # The CombinedProcessor will consolidate all the parsed data into this variable
     combined_maintenance_data: Dict = {}
 
+    def process(self, data: NotificationData, extended_data: Dict) -> Iterable[Maintenance]:
+        """Extend base class process method to ensure that self.combined_maintenance_data is initialized correctly."""
+        self.combined_maintenance_data = {}
+        return super().process(data, extended_data)
+
     def process_hook(self, maintenances_extracted_data, maintenances_data):
         """All the parsers contribute with a subset of data that is extended.
 

circuit_maintenance_parser/provider.py

@@ -1,8 +1,9 @@
 """Definition of Provider class as the entry point to the library."""
 import logging
+import re
 import traceback
 
-from typing import Iterable, List
+from typing import Iterable, List, Dict
 
 from pydantic import BaseModel
 
@@ -13,8 +14,10 @@
 from circuit_maintenance_parser.parser import ICal, EmailDateParser
 from circuit_maintenance_parser.errors import ProcessorError, ProviderError
 from circuit_maintenance_parser.processor import CombinedProcessor, SimpleProcessor, GenericProcessor
+from circuit_maintenance_parser.constants import EMAIL_HEADER_SUBJECT
 
 from circuit_maintenance_parser.parsers.aquacomms import HtmlParserAquaComms1, SubjectParserAquaComms1
+from circuit_maintenance_parser.parsers.aws import SubjectParserAWS1, TextParserAWS1
 from circuit_maintenance_parser.parsers.cogent import HtmlParserCogent1
 from circuit_maintenance_parser.parsers.colt import ICalParserColt1, CsvParserColt1
 from circuit_maintenance_parser.parsers.gtt import HtmlParserGTT1
@@ -49,6 +52,14 @@ class GenericProvider(BaseModel):
             that will be used. Default: `[SimpleProcessor(data_parsers=[ICal])]`.
         _default_organizer (optional): Defines a default `organizer`, an email address, to be used to create a
             `Maintenance` in absence of the information in the original notification.
+        _include_filter (optional): Dictionary that defines matching regex per data type to take a notification into
+            account.
+        _exclude_filter (optional): Dictionary that defines matching regex per data type to NOT take a notification
+            into account.
+
+    Notes:
+        - If a notification matches both the `_include_filter` and `_exclude_filter`, the exclusion takes precedence and
+          the notification will be filtered out.
 
     Examples:
         >>> GenericProvider()
@@ -58,12 +69,55 @@ class GenericProvider(BaseModel):
     _processors: List[GenericProcessor] = [SimpleProcessor(data_parsers=[ICal])]
     _default_organizer: str = "unknown"
 
+    _include_filter: Dict[str, List[str]] = {}
+    _exclude_filter: Dict[str, List[str]] = {}
+
+    def include_filter_check(self, data: NotificationData) -> bool:
+        """If `_include_filter` is defined, it verifies that the matching criteria is met."""
+        if self._include_filter:
+            return self.filter_check(self._include_filter, data, "include")
+        return True
+
+    def exclude_filter_check(self, data: NotificationData) -> bool:
+        """If `_exclude_filter` is defined, it verifies that the matching criteria is met."""
+        if self._exclude_filter:
+            return self.filter_check(self._exclude_filter, data, "exclude")
+        return False
+
+    @staticmethod
+    def filter_check(filter_dict: Dict, data: NotificationData, filter_type: str) -> bool:
+        """Generic filter check."""
+        data_part_content = None
+        for data_part in data.data_parts:
+            filter_data_type = data_part.type
+            if filter_data_type not in filter_dict:
+                continue
+
+            data_part_content = data_part.content.decode()
+            if any(re.search(filter_re, data_part_content) for filter_re in filter_dict[filter_data_type]):
+                logger.debug("Matching %s filter expression for %s.", filter_type, data_part_content)
+                return True
+
+        if data_part_content:
+            logger.warning("Not matching any %s filter expression for %s.", filter_type, data_part_content)
+        else:
+            logger.warning(
+                "Not matching any %s filter expression because the notification doesn't contain the expected data_types: %s",
+                filter_type,
+                ", ".join(filter_dict.keys()),
+            )
+        return False
+
     def get_maintenances(self, data: NotificationData) -> Iterable[Maintenance]:
         """Main entry method that will use the defined `_processors` in order to extract the `Maintenances` from data."""
         provider_name = self.__class__.__name__
         error_message = ""
         related_exceptions = []
 
+        if self.exclude_filter_check(data) or not self.include_filter_check(data):
+            logger.debug("Skipping notification %s due filtering policy for %s.", data, self.__class__.__name__)
+            return []
+
         for processor in self._processors:
             try:
                 return processor.process(data, self.get_extended_data())
@@ -116,6 +170,15 @@ class AquaComms(GenericProvider):
     _default_organizer = "[email protected]"
 
 
+class AWS(GenericProvider):
+    """AWS provider custom class."""
+
+    _processors: List[GenericProcessor] = [
+        CombinedProcessor(data_parsers=[EmailDateParser, TextParserAWS1, SubjectParserAWS1]),
+    ]
+    _default_organizer = "[email protected]"
+
+
 class Cogent(GenericProvider):
     """Cogent provider custom class."""
 
@@ -162,6 +225,8 @@ class HGC(GenericProvider):
 class Lumen(GenericProvider):
     """Lumen provider custom class."""
 
+    _include_filter = {EMAIL_HEADER_SUBJECT: ["Scheduled Maintenance"]}
+
     _processors: List[GenericProcessor] = [
         CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserLumen1]),
     ]