Merge pull request #125 from networktocode/gfm-issue-115

Make 'status' optional, accept iCal notifications with no 'sequence' either

Author: glennmatthews

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Fixed
 
+- #115 - Add default `status` and `sequence` values for iCal notifications missing these fields
 - #124 - Handle encoded non-ASCII characters in email subjects.
 - #126 - Ignore a class of non-maintenance-notification emails from Telia.
 

README.md

@@ -25,14 +25,24 @@ You can leverage this library in your automation framework to process circuit ma
 - **account**: identifies an account associated with the service that is the subject of the maintenance notification.
 - **maintenance_id**: contains text that uniquely identifies the maintenance that is the subject of the notification.
 - **circuits**: list of circuits affected by the maintenance notification and their specific impact.
-- **status**: defines the overall status or confirmation for the maintenance.
 - **start**: timestamp that defines the start date of the maintenance in GMT.
 - **end**: timestamp that defines the end date of the maintenance in GMT.
 - **stamp**: timestamp that defines the update date of the maintenance in GMT.
 - **organizer**: defines the contact information included in the original notification.
 
+and may additionally contain the following attributes:
+
+- **status**: defines the overall status or confirmation for the maintenance.¹
+- **sequence**: a sequence number for notifications involving this maintenance window. In practice this is generally redundant with the **stamp** field, and will be defaulted to `1` for most non-iCal parsed notifications.²
+- **summary**: human-readable details about this maintenance notification.
+- **uid**: a unique (?) identifer for a thread of related notifications. In practice this is generally redundant with the **maintenance_id** field, and will be defaulted to `0` for most non-iCal parsed notifications.
+
 > Please, refer to the [BCOP](https://github.com/jda/maintnote-std/blob/master/standard.md) to more details about these attributes.
 
+¹ Per the BCOP, **status** (`X-MAINTNOTE_STATUS`) is an optional field in notifications. However, a `Maintenance` object will always contain a `status` value; in the case where the source notification omits this field, the `status` will be set to `"NO-CHANGE"`, and it's up to the consumer of this library to determine how to appropriately handle this case.
+
+² Per the BCOP, **sequence** is a mandatory field in notifications. However, some NSPs have been seen to send notifications which, while otherwise consistent with the BCOP, omit the `SEQUENCE` field; in such cases, this library will report a sequence number of `-1`.
+
 ## Workflow
 
 1. We instantiate a `Provider`, directly or via the `init_provider` method, that depending on the selected type will return the corresponding instance.

circuit_maintenance_parser/output.py

@@ -37,8 +37,9 @@ class Status(str, Enum):
     - "CONFIRMED": Indicates maintenance event is definite.
     - "CANCELLED": Indicates maintenance event was cancelled.
     - "IN-PROCESS": Indicates maintenance event is in process (e.g. open).
-    - "COMPLETED":Indicates maintenance event completed (e.g. closed).
+    - "COMPLETED": Indicates maintenance event completed (e.g. closed).
     - "RE-SCHEDULED": Indicates maintenance event was re-scheduled.
+    - "NO-CHANGE": Indicates status is unchanged from a previous notification (dummy value)
     """
 
     TENTATIVE = "TENTATIVE"
@@ -48,6 +49,8 @@ class Status(str, Enum):
     COMPLETED = "COMPLETED"
     RE_SCHEDULED = "RE-SCHEDULED"
 
+    NO_CHANGE = "NO-CHANGE"
+
 
 class CircuitImpact(BaseModel, extra=Extra.forbid):
     """CircuitImpact class.
@@ -96,13 +99,13 @@ class Maintenance(BaseModel, extra=Extra.forbid):
         account:  identifies an account associated with the service that is the subject of the maintenance notification
         maintenance_id:  contains text that uniquely identifies the maintenance that is the subject of the notification
         circuits: list of circuits affected by the maintenance notification and their specific impact
-        status: defines the overall status or confirmation for the maintenance
         start: timestamp that defines the start date of the maintenance in GMT
         end: timestamp that defines the end date of the maintenance in GMT
         stamp: timestamp that defines the update date of the maintenance in GMT
         organizer: defines the contact information included in the original notification
 
     Optional attributes:
+        status: defines the overall status or confirmation for the maintenance
         summary: description of the maintenace notification
         uid: specific unique identifier for each notification
         sequence: sequence number - initially zero - to serialize updates in case they are received or processed out of
@@ -123,20 +126,20 @@ class Maintenance(BaseModel, extra=Extra.forbid):
         ...     summary="This is a maintenance notification",
         ...     uid="1111",
         ... )
-        Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', circuits=[CircuitImpact(circuit_id='123', impact=<Impact.NO_IMPACT: 'NO-IMPACT'>), CircuitImpact(circuit_id='456', impact=<Impact.OUTAGE: 'OUTAGE'>)], status=<Status.COMPLETED: 'COMPLETED'>, start=1533704400, end=1533712380, stamp=1533595768, organizer='[email protected]', uid='1111', sequence=1, summary='This is a maintenance notification')
+        Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', circuits=[CircuitImpact(circuit_id='123', impact=<Impact.NO_IMPACT: 'NO-IMPACT'>), CircuitImpact(circuit_id='456', impact=<Impact.OUTAGE: 'OUTAGE'>)], start=1533704400, end=1533712380, stamp=1533595768, organizer='[email protected]', status=<Status.COMPLETED: 'COMPLETED'>, uid='1111', sequence=1, summary='This is a maintenance notification')
     """
 
     provider: StrictStr
     account: StrictStr
     maintenance_id: StrictStr
     circuits: List[CircuitImpact]
-    status: Status
     start: StrictInt
     end: StrictInt
     stamp: StrictInt
     organizer: StrictStr
 
     # Non mandatory attributes
+    status: Status = Status.NO_CHANGE
     uid: StrictStr = "0"
     sequence: StrictInt = 1
     summary: StrictStr = ""

circuit_maintenance_parser/parser.py

@@ -110,14 +110,18 @@ def parse_ical(gcal: Calendar) -> List[Dict]:
                     "provider": str(component.get("X-MAINTNOTE-PROVIDER")),
                     "account": str(component.get("X-MAINTNOTE-ACCOUNT")),
                     "maintenance_id": str(component.get("X-MAINTNOTE-MAINTENANCE-ID")),
-                    "status": Status(component.get("X-MAINTNOTE-STATUS")),
+                    # status may be omitted, per the BCOP
+                    "status": Status(component.get("X-MAINTNOTE-STATUS", "NO-CHANGE")),
                     "start": round(component.get("DTSTART").dt.timestamp()),
                     "end": round(component.get("DTEND").dt.timestamp()),
                     "stamp": round(component.get("DTSTAMP").dt.timestamp()),
                     "summary": str(component.get("SUMMARY")),
                     "organizer": str(component.get("ORGANIZER")),
                     "uid": str(component.get("UID")),
-                    "sequence": int(component.get("SEQUENCE")),
+                    # per the BCOP sequence is mandatory, but we have real examples where it's omitted,
+                    # usually in combination with an omitted status. In that case let's treat the sequence as -1,
+                    # i.e. older than all known updates.
+                    "sequence": int(component.get("SEQUENCE", -1)),
                 }
 
                 data = {key: value for key, value in data.items() if value != "None"}

tests/unit/data/ical/ical6

@@ -0,0 +1,20 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:Data::ICal 0.22
+BEGIN:VEVENT
+DESCRIPTION:NTT (AS2914) Regular Maintenance Notification: completed [GIN-C
+ HG0123456]\nPlease refer to the email notification for more details regard
+ ing maintenance or outage reason and impact.
+DTEND:20211115T205700Z
+DTSTAMP:20211115T205904Z
+DTSTART:20211115T180000Z
+SUMMARY:NTT (AS2914) Regular Maintenance Notification: completed [GIN-CHG01
+ 23456]
+UID:GIN-CHG0040894
+X-MAINTNOTE-ACCOUNT:47004321
+X-MAINTNOTE-IMPACT:OUTAGE
+X-MAINTNOTE-MAINTENANCE-ID:GIN-CHG0123456
+X-MAINTNOTE-OBJECT-ID;X-MAINTNOTE-OBJECT-IMPACT=OUTAGE:246810
+X-MAINTNOTE-PROVIDER:NTT (AS2914)
+END:VEVENT
+END:VCALENDAR

tests/unit/data/ical/ical6_result.json

@@ -0,0 +1,20 @@
+[
+    {
+        "account": "47004321",
+        "circuits": [
+            {
+                "circuit_id": "246810",
+                "impact": "OUTAGE"
+            }
+        ],
+        "end": 1637009820,
+        "maintenance_id": "GIN-CHG0123456",
+        "provider": "NTT (AS2914)",
+        "sequence": -1,
+        "stamp": 1637009944,
+        "start": 1636999200,
+        "status": "NO-CHANGE",
+        "summary": "NTT (AS2914) Regular Maintenance Notification: completed [GIN-CHG0123456]",
+        "uid": "GIN-CHG0040894"
+    }
+]

tests/unit/test_output.py

@@ -35,11 +35,11 @@
         ("provider", None, ValidationError),
         ("provider", 1, ValidationError),
         ("provider", "good", None),
-        ("status", None, ValidationError),
+        # Non mandatory attributes
+        ("status", None, None),
         ("status", 1, ValidationError),
         ("status", "good", ValidationError),
         ("status", "IN-PROCESS", None),
-        # Non mandatory attributes
         ("sequence", None, None),
         ("sequence", "1", ValidationError),
         ("sequence", 1, None),

tests/unit/test_parsers.py

@@ -43,6 +43,7 @@
         (ICal, Path(dir_path, "data", "ical", "ical3"), Path(dir_path, "data", "ical", "ical3_result.json"),),
         (ICal, Path(dir_path, "data", "ical", "ical4"), Path(dir_path, "data", "ical", "ical4_result.json"),),
         (ICal, Path(dir_path, "data", "ical", "ical5"), Path(dir_path, "data", "ical", "ical5_result.json"),),
+        (ICal, Path(dir_path, "data", "ical", "ical6"), Path(dir_path, "data", "ical", "ical6_result.json"),),
         # AquaComms
         (
             HtmlParserAquaComms1,