Merge pull request #6

* Support for saving summary in multiple formats.

* Support for parsing reported NAV for each scheme.

Author: codereverser

README.md

@@ -63,6 +63,7 @@ data = casparser.read_cas_pdf('/path/to/cas/pdf/file.pdf', 'password')
                     "close": "number",
                     "valuation": {
                       "date": "date",
+                      "nav": "number",
                       "value": "number"
                     },
                     "transactions": [

assets/demo.jpg

@@ -1,108 +1,178 @@
 from decimal import Decimal
 import json
+import os
 import re
+import shutil
 import sys
+import textwrap
 
 import click
-import texttable
+
+# noinspection PyProtectedMember
+from tabulate import tabulate, _table_formats
+
 
 from .__version__ import __version__
 
 from . import read_cas_pdf
 from .encoder import CASDataEncoder
 from .exceptions import ParserException
+from .parsers.utils import isclose
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
 
-# noinspection PyUnusedLocal
-def validate_output_filename(ctx, param, filename: str):
-    """Validate whether output filename ends with json."""
-    if filename is None or filename.lower().endswith(".json"):
-        return filename
-    raise click.BadParameter("Output filename should end with .json")
-
-
-def print_summary(data):
+def print_summary(data, tablefmt="fancy_grid", output_filename=None, include_zero_folios=False):
     """Print summary of parsed data."""
     count = 0
     err = 0
-    click.echo("\n")
-    click.echo(
-        f"{'Statement Period':>40s}: "
-        f"{click.style(data['statement_period']['from'], fg='green', bold=True)}"
-        f"  To {click.style(data['statement_period']['to'], fg='green', bold=True)}"
-    )
-    click.echo(f"{'File Type':>40s}: {click.style(data['file_type'], bold=True)}")
-    for key, value in data["investor_info"].items():
-        fmt_value = " ".join([x.strip() for x in value.splitlines()])
-        fmt_value = re.sub(r"\s+", " ", fmt_value)
-        if len(fmt_value) > 40:
-            fmt_value = fmt_value[:37] + "..."
-        click.echo(f"{key:>40s}: {fmt_value}")
-    click.echo("")
-    table = texttable.Texttable(max_width=120)
-    header = [
+
+    if output_filename:
+        fmt = "fancy_grid"
+    else:
+        fmt = tablefmt
+
+    print_extra_info = fmt in ("simple", "plain", "fancy_grid", "grid", "pretty")
+    if print_extra_info:
+        click.echo("\n")
+        click.echo(
+            f"{'Statement Period':>40s}: "
+            f"{click.style(data['statement_period']['from'], fg='green', bold=True)}"
+            f"  To {click.style(data['statement_period']['to'], fg='green', bold=True)}"
+        )
+        click.echo(f"{'File Type':>40s}: {click.style(data['file_type'], bold=True)}")
+        for key, value in data["investor_info"].items():
+            fmt_value = " ".join([x.strip() for x in value.splitlines()])
+            fmt_value = re.sub(r"\s+", " ", fmt_value)
+            if len(fmt_value) > 40:
+                fmt_value = fmt_value[:37] + "..."
+            click.echo(f"{key:>40s}: {fmt_value}")
+        click.echo("")
+    rows = []
+    console_rows = []
+
+    console_header = [
         "Scheme",
         "Open",
-        "Close\nReported",
-        "Close\nCalculated",
+        "Close\n\nReported\nvs.\nCalculated",
         f"Value\n({data['statement_period']['to']})",
+        "Txns",
+        "",
+    ]
+    header = [
+        "Scheme",
+        "Open",
+        "Close Reported",
+        "Close Calculated",
+        f"NAV ({data['statement_period']['to']})",
+        f"Value ({data['statement_period']['to']})",
         "Transactions",
         "Status",
     ]
-    table.add_row(header)
-    table.set_cols_align(["l"] + ["r"] * (len(header) - 2) + ["c"])
-    table.set_cols_valign(["m"] * len(header))
+    col_align = ["left"] + ["right"] * (len(header) - 2) + ["center"]
+    console_col_align = ["left"] + ["right"] * (len(console_header) - 2) + ["center"]
+
     current_amc = None
     value = Decimal(0)
+    columns, _ = shutil.get_terminal_size()
+    scheme_col_width = columns - 66
+
+    folio_header_added = False
     for folio in data["folios"]:
         if current_amc != folio.get("amc", ""):
+            folio_header_added = False
             current_amc = folio["amc"]
-            table.add_row([current_amc] + [""] * 6)
         for scheme in folio["schemes"]:
+
+            if scheme["close"] < 1e-3 and not include_zero_folios:
+                continue
+
             calc_close = scheme["open"] + sum([x["units"] for x in scheme["transactions"]])
             valuation = scheme["valuation"]
-            if calc_close != scheme["close"]:
+
+            # Check is calculated close (i.e. open + units from all transactions) is same as
+            # reported close and also the scheme valuation = nav * calculated close.
+            if calc_close != scheme["close"] or not isclose(
+                valuation["nav"] * calc_close, valuation["value"], tol=2
+            ):
                 err += 1
                 status = "❗️"
             else:
                 status = "️✅"
-            scheme_name = f"{scheme['scheme']}\nFolio: {folio['folio']}"
+            wrapped_name = textwrap.fill(scheme["scheme"], width=scheme_col_width)
+            folio_number = re.sub(r"\s+", "", folio["folio"])
+            folio_string = textwrap.fill(f"Folio: {folio_number}", width=scheme_col_width)
+            scheme_name = f"{wrapped_name}\n{folio_string}"
             value += valuation["value"]
-            table.add_row(
+
+            if not folio_header_added:
+                rows.append(
+                    [textwrap.fill(current_amc, width=scheme_col_width)] + [""] * (len(header) - 1)
+                )
+                console_rows.append(
+                    [textwrap.fill(current_amc, width=scheme_col_width)]
+                    + [""] * (len(console_header) - 1)
+                )
+                folio_header_added = True
+
+            console_rows.append(
+                [
+                    scheme_name,
+                    scheme["open"],
+                    f"{scheme['close']}\n/\n{calc_close}",
+                    f"₹{valuation['value']:,.2f}\n@\n₹{valuation['nav']:,.2f}",
+                    len(scheme["transactions"]),
+                    status,
+                ]
+            )
+            rows.append(
                 [
                     scheme_name,
                     scheme["open"],
                     scheme["close"],
                     calc_close,
-                    f"₹{valuation['value']:,.2f}",
+                    valuation["nav"],
+                    valuation["value"],
                     len(scheme["transactions"]),
                     status,
                 ]
             )
             count += 1
-    click.echo(table.draw())
-    click.echo(
-        "Portfolio Valuation : "
-        + click.style(f"₹{value:,.2f}", fg="green", bold=True)
-        + f" [As of {data['statement_period']['to']}]"
-    )
-    click.secho("Summary", bold=True)
-    click.echo("Total   : " + click.style(f"{count:4d}", fg="white", bold=True) + " schemes")
-    click.echo("Matched : " + click.style(f"{count - err:4d}", fg="green", bold=True) + " schemes")
-    click.echo("Error   : " + click.style(f"{err:4d}", fg="red", bold=True) + " schemes")
+    if print_extra_info:
+        click.echo(tabulate(console_rows, console_header, tablefmt=fmt, colalign=console_col_align))
+        click.echo(
+            "Portfolio Valuation : "
+            + click.style(f"₹{value:,.2f}", fg="green", bold=True)
+            + f" [As of {data['statement_period']['to']}]"
+        )
+        click.secho("Summary", bold=True)
+        click.echo("Total   : " + click.style(f"{count:4d}", fg="white", bold=True) + " schemes")
+        click.echo(
+            "Matched : " + click.style(f"{count - err:4d}", fg="green", bold=True) + " schemes"
+        )
+        click.echo("Error   : " + click.style(f"{err:4d}", fg="red", bold=True) + " schemes")
+    else:
+        click.echo(tabulate(rows, header, tablefmt=fmt, colalign=col_align))
+
+    if output_filename:
+        with open(output_filename, "w") as f:
+            f.write(tabulate(rows, header, tablefmt=tablefmt, colalign=col_align))
+        click.echo("File saved : " + click.style(output_filename, bold=True))
 
 
 @click.command(name="casparser", context_settings=CONTEXT_SETTINGS)
 @click.option(
     "-o",
     "--output",
-    help="Output file path (json)",
-    callback=validate_output_filename,
-    type=click.Path(exists=False, dir_okay=False, writable=True),
+    help="Output file path",
+    type=click.Path(dir_okay=False, writable=True),
+)
+@click.option(
+    "-s",
+    "--summary",
+    type=click.Choice(_table_formats.keys()),
+    help="Print Summary of transactions parsed.",
 )
-@click.option("-s", "--summary", is_flag=True, help="Print Summary of transactions parsed.")
 @click.option(
     "-p",
     "password",
@@ -112,24 +182,36 @@ def print_summary(data):
     confirmation_prompt=False,
     help="CAS password",
 )
+@click.option(
+    "-a",
+    "--include-all",
+    is_flag=True,
+    help="Include schemes with zero valuation in the summary output"
+)
 @click.option(
     "--force-pdfminer", is_flag=True, help="Force PDFMiner parser even if MuPDF is detected"
 )
 @click.version_option(__version__, prog_name="casparser-cli")
 @click.argument("filename", type=click.Path(exists=True), metavar="CAS_PDF_FILE")
-def cli(output, summary, password, force_pdfminer, filename):
+def cli(output, summary, password, include_all, force_pdfminer, filename):
     """CLI function."""
-    if output is None and not summary:
-        click.echo("No output file provided. Printing summary")
-        summary = True
+    output_ext = None
+    if output is not None:
+        output_ext = os.path.splitext(output)[-1].lower()
+
+    if not (summary or output_ext == ".json"):
+        summary = "fancy_grid"
+
     try:
         data = read_cas_pdf(filename, password, force_pdfminer=force_pdfminer)
     except ParserException as exc:
         click.echo("Error parsing pdf file :: " + click.style(str(exc), bold=True, fg="red"))
         sys.exit(1)
     if summary:
-        print_summary(data)
-    if output is not None:
+        print_summary(
+            data, tablefmt=summary, include_zero_folios=include_all, output_filename=None if output_ext == ".json" else output
+        )
+    if output_ext == ".json":
         with open(output, "w") as fp:
             json.dump(data, fp, cls=CASDataEncoder, indent=2)
         click.echo("File saved : " + click.style(output, bold=True))

casparser/cli.py

@@ -5,7 +5,7 @@
 
 from .exceptions import HeaderParseError, CASParseError
 from .regex import FOLIO_RE, HEADER_RE, SCHEME_RE
-from .regex import CLOSE_UNITS_RE, OPEN_UNITS_RE, VALUATION_RE, TRANSACTION_RE
+from .regex import CLOSE_UNITS_RE, NAV_RE, OPEN_UNITS_RE, VALUATION_RE, TRANSACTION_RE
 
 
 def parse_header(text):
@@ -64,7 +64,7 @@ def process_cas_text(text):
                     "rta": m.group(4).strip(),
                     "open": Decimal(0.0),
                     "close": Decimal(0.0),
-                    "valuation": {"date": None, "value": 0},
+                    "valuation": {"date": None, "value": 0, "nav": 0},
                     "transactions": [],
                 }
         if not curr_scheme_data:
@@ -79,6 +79,11 @@ def process_cas_text(text):
                 date=date_parser.parse(m.group(1)).date(),
                 value=Decimal(m.group(2).replace(",", "_")),
             )
+        if m := re.search(NAV_RE, line, re.I):
+            curr_scheme_data["valuation"].update(
+                date=date_parser.parse(m.group(1)).date(),
+                nav=Decimal(m.group(2).replace(",", "_")),
+            )
             continue
         if m := re.search(TRANSACTION_RE, line, re.DOTALL):
             date = date_parser.parse(m.group(1)).date()

casparser/process.py

@@ -9,6 +9,7 @@
 OPEN_UNITS_RE = r"Opening\s+Unit\s+Balance.+?([\d,.]+)"
 CLOSE_UNITS_RE = r"Closing\s+Unit\s+Balance.+?([\d,.]+)"
 VALUATION_RE = r"Valuation\s+on\s+(\d{2}-[A-Za-z]{3}-\d{4})\s*:\s*INR\s*([\d,.]+)"
+NAV_RE = r"NAV\s+on\s+(\d{2}-[A-Za-z]{3}-\d{4})\s*:\s*INR\s*([\d,.]+)"
 
 TRANSACTION_RE = (
     r"(\d{2}-[A-Za-z]{3}-\d{4})\t\t([^\t]+?)\t\t([(\d,.]+)\)*\t\t"

casparser/regex.py

@@ -31,6 +31,7 @@ class SchemeValuationType(TypedDict):
     """Scheme valuation as of a given date."""
 
     date: Union[date, str]
+    nav: Union[Decimal, float]
     value: Union[Decimal, float]
 
 

casparser/types.py

@@ -2,5 +2,5 @@ click==7.1.2
 colorama==0.4.4
 pdfminer.six==20201018
 python-dateutil==2.8.1
-texttable==1.6.3
+tabulate==0.8.7
 wcwidth==0.2.5

requirements.txt

@@ -24,9 +24,9 @@ install_requires =
     click>=7.1.2
     colorama>=0.4.4
     pdfminer.six>=20201018
-    python-dateutil==2.8.1
-    texttable>=1.6.3
-    wcwidth==0.2.5
+    python-dateutil>=2.8.1
+    tabulate>=0.8.7
+    wcwidth>=0.2.5
 
 [options.entry_points]
 console_scripts =

setup.cfg

@@ -26,18 +26,24 @@ def test_cli(self, tmpdir):
         assert result.exit_code == 0
         assert "File saved" in result.output
 
-        fpath = tmpdir.join("output.txt")
+        fpath = tmpdir.join("output.html")
         result = runner.invoke(
-            cli, [self.cams_file_name, "-p", self.cams_password, "-o", fpath.strpath]
+            cli, [self.cams_file_name, "-p", self.cams_password, "-o", fpath.strpath, "-s", "html"]
+        )
+        assert result.exit_code != 1
+        assert "File saved" in result.output
+
+        result = runner.invoke(
+            cli, [self.cams_file_name, "-p", self.cams_password, "-s", "html"]
         )
         assert result.exit_code != 1
-        assert "Output filename should end" in result.output
+        assert "<table>\n<thead>" in result.output
 
         result = runner.invoke(cli, [self.kfintech_file_name, "-p", self.cams_password])
         assert result.exit_code != 0
         assert "Incorrect PDF password!" in result.output
 
-        result = runner.invoke(cli, [self.bad_file_name, "-p", ""])
+        result = runner.invoke(cli, [self.bad_file_name, "-p", "", "-a"])
         assert result.exit_code == 0
         assert re.search(r"Error\s+:\s+1\s+schemes", result.output) is not None