Support for parsing dividend payouts (#12)

* Support for parsing dividend payout transactions

* code formatting (black)

* fixed cli calculation

* add support for identifying dividend reinvestments

* v0.3.7 version bump

Author: codereverser

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.7 - 2020-12-24
+
+- Support for parsing dividend transactions
+
 ## 0.3.6 - 2020-12-21
 
 - Support for parsing folios without advisor

README.md

@@ -72,7 +72,10 @@ data = casparser.read_cas_pdf('/path/to/cas/pdf/file.pdf', 'password')
                             "amount": "number",
                             "units": "number",
                             "nav": "number",
-                            "balance": "number"
+                            "balance": "number",
+                            "is_dividend_payout": "boolean",
+                            "is_dividend_reinvestment": "boolean",
+                            "dividend_rate": null
                         }
                     ]
                 }

casparser/VERSION.txt

@@ -1 +1 @@
-0.3.6
+0.3.7

casparser/cli.py

@@ -87,7 +87,9 @@ def print_summary(data, tablefmt="fancy_grid", output_filename=None, include_zer
             if scheme["close"] < 1e-3 and not include_zero_folios:
                 continue
 
-            calc_close = scheme["open"] + sum([x["units"] for x in scheme["transactions"]])
+            calc_close = scheme["open"] + sum(
+                [x["units"] for x in scheme["transactions"] if x["units"] is not None]
+            )
             valuation = scheme["valuation"]
 
             # Check is calculated close (i.e. open + units from all transactions) is same as
@@ -186,7 +188,7 @@ def print_summary(data, tablefmt="fancy_grid", output_filename=None, include_zer
     "-a",
     "--include-all",
     is_flag=True,
-    help="Include schemes with zero valuation in the summary output"
+    help="Include schemes with zero valuation in the summary output",
 )
 @click.option(
     "--force-pdfminer", is_flag=True, help="Force PDFMiner parser even if MuPDF is detected"
@@ -209,7 +211,10 @@ def cli(output, summary, password, include_all, force_pdfminer, filename):
         sys.exit(1)
     if summary:
         print_summary(
-            data, tablefmt=summary, include_zero_folios=include_all, output_filename=None if output_ext == ".json" else output
+            data,
+            tablefmt=summary,
+            include_zero_folios=include_all,
+            output_filename=None if output_ext == ".json" else output,
         )
     if output_ext == ".json":
         with open(output, "w") as fp:

casparser/parsers/pdfminer.py

@@ -46,7 +46,9 @@ def parse_investor_info(layout, width, height) -> InvestorInfo:
                 mobile = m.group(1).strip()
             address_lines.append(txt)
             if mobile is not None:
-                return InvestorInfo(email=email, name=name, mobile=mobile, address="\n".join(address_lines))
+                return InvestorInfo(
+                    email=email, name=name, mobile=mobile, address="\n".join(address_lines)
+                )
     if email is None or mobile is None:
         raise CASParseError("Unable to parse investor data")
 

casparser/process.py

@@ -5,7 +5,8 @@
 
 from .exceptions import HeaderParseError, CASParseError
 from .regex import FOLIO_RE, HEADER_RE, SCHEME_RE
-from .regex import CLOSE_UNITS_RE, NAV_RE, OPEN_UNITS_RE, VALUATION_RE, TRANSACTION_RE
+from .regex import CLOSE_UNITS_RE, NAV_RE, OPEN_UNITS_RE, VALUATION_RE
+from .regex import DESCRIPTION_TAIL_RE, DIVIDEND_RE, TRANSACTION_RE
 
 
 def parse_header(text):
@@ -31,8 +32,14 @@ def process_cas_text(text):
     current_folio = None
     current_amc = None
     curr_scheme_data = {}
+    balance = Decimal(0.0)
     lines = text.split("\u2029")
     for line in lines:
+        if m := re.search(DESCRIPTION_TAIL_RE, line, re.I | re.DOTALL):
+            description_tail = m.group(1).rstrip()
+            line = line.replace(description_tail, "")
+        else:
+            description_tail = ""
         if amc_match := re.search(r"^(.+?)\s+(MF|Mutual\s+Fund)$", line, re.I | re.DOTALL):
             current_amc = amc_match.group(0)
         elif m := re.search(FOLIO_RE, line, re.I):
@@ -70,6 +77,7 @@ def process_cas_text(text):
                     "valuation": {"date": None, "value": 0, "nav": 0},
                     "transactions": [],
                 }
+                balance = Decimal(0.0)
         if not curr_scheme_data:
             continue
         if m := re.search(OPEN_UNITS_RE, line):
@@ -90,11 +98,23 @@ def process_cas_text(text):
             continue
         if m := re.search(TRANSACTION_RE, line, re.DOTALL):
             date = date_parser.parse(m.group(1)).date()
+            desc = m.group(2).strip() + description_tail
             amt = Decimal(m.group(3).replace(",", "_").replace("(", "-"))
-            units = Decimal(m.group(4).replace(",", "_").replace("(", "-"))
-            nav = Decimal(m.group(5).replace(",", "_"))
-            balance = Decimal(m.group(6).replace(",", "_"))
-            desc = m.group(2).strip()
+            if m.group(4) is None:
+                units = None
+                nav = None
+            else:
+                units = Decimal(m.group(4).replace(",", "_").replace("(", "-"))
+                nav = Decimal(m.group(5).replace(",", "_"))
+                balance = Decimal(m.group(6).replace(",", "_"))
+            if div_match := re.search(DIVIDEND_RE, desc, re.I | re.DOTALL):
+                reinvest_flag, rate = div_match.groups()
+                is_dividend_payout = reinvest_flag is None
+                is_dividend_reinvestment = not is_dividend_payout
+                dividend_rate = Decimal(rate)
+            else:
+                is_dividend_payout = is_dividend_reinvestment = False
+                dividend_rate = None
             curr_scheme_data["transactions"].append(
                 {
                     "date": date,
@@ -103,6 +123,9 @@ def process_cas_text(text):
                     "units": units,
                     "nav": nav,
                     "balance": balance,
+                    "is_dividend_payout": is_dividend_payout,
+                    "is_dividend_reinvestment": is_dividend_reinvestment,
+                    "dividend_rate": dividend_rate,
                 }
             )
     if curr_scheme_data:

casparser/regex.py

@@ -12,6 +12,9 @@
 NAV_RE = r"NAV\s+on\s+(\d{2}-[A-Za-z]{3}-\d{4})\s*:\s*INR\s*([\d,.]+)"
 
 TRANSACTION_RE = (
-    r"(\d{2}-[A-Za-z]{3}-\d{4})\t\t([^\t]+?)\t\t([(\d,.]+)\)*\t\t"
-    r"([(\d,.]+)\)*\t\t([(\d,.]+)\)*\t\t([(\d,.]+)\)*"
+    # r"(\d{2}-[A-Za-z]{3}-\d{4})\t\t([^\t]+?)\t\t([(\d,.]+)\)*\t\t([(\d,.]+)\)*\t\t([(\d,.]+)\)*\t\t([(\d,.]+)\)*"
+    r"(\d{2}-[A-Za-z]{3}-\d{4})\t\t([^\t]+?)\t\t([(\d,.]+)\)*(?:\t\t([(\d,.]+)\)*\t\t([(\d,.]+)\)*\t\t([(\d,.]+)\)*)*"
 )
+DIVIDEND_RE = r"dividend.+?(reinvest)*.+?@\s+Rs\.\s*([\d\.]+)\s+per\s+unit"
+
+DESCRIPTION_TAIL_RE = r"\d{2}-[A-Za-z]{3}-\d{4}\t\t.*(\n[^\t]+)[\t|$]"

casparser/types.py

@@ -22,9 +22,12 @@ class TransactionType(TypedDict):
     date: Union[date, str]
     description: str
     amount: Union[Decimal, float]
-    units: Union[Decimal, float]
-    nav: Union[Decimal, float]
+    units: Union[Decimal, float, None]
+    nav: Union[Decimal, float, None]
     balance: Union[Decimal, float]
+    is_dividend_payout: bool
+    is_dividend_reinvestment: bool
+    dividend_rate: Union[Decimal, float, None]
 
 
 class SchemeValuationType(TypedDict):

tests/files.enc

@@ -33,9 +33,7 @@ def test_cli(self, tmpdir):
         assert result.exit_code != 1
         assert "File saved" in result.output
 
-        result = runner.invoke(
-            cli, [self.cams_file_name, "-p", self.cams_password, "-s", "html"]
-        )
+        result = runner.invoke(cli, [self.kfintech_file_name, "-p", self.kfintech_password, "-s", "html"])
         assert result.exit_code != 1
         assert "<table>\n<thead>" in result.output