Merge pull request #13 from codereverser/feature/transaction-types

0.3.9 release - Transaction classification support + more...

- Transaction classification
- csv output support

Author: codereverser

.codecov.yml

@@ -1,7 +1,7 @@
 coverage:
   precision: 2
   round: down
-  range: "70...90"
+  range: "80...90"
   status:
     project:
       default:

.deepsource.toml

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.9 - 2021-01-01
+
+- Support for classifying  transactions
+
 ## 0.3.8 - 2020-12-29
 
 - Support for parsing folios without PAN/KYC details
@@ -55,7 +59,7 @@
 
 ## 0.2.0 - 2020-10-15
 
-- removed support for python < 3.8 versions
+- removed support for python versions <3.8
 - Better investor info parser
 
 ## 0.1.2 - 2020-10-14

CHANGELOG.md

@@ -25,9 +25,16 @@ pip install casparser[mupdf]
 
 ## Usage
 
-```
+```python
 import casparser
-data = casparser.read_cas_pdf('/path/to/cas/pdf/file.pdf', 'password')
+data = casparser.read_cas_pdf("/path/to/cas/file.pdf", "password")
+
+# Get data in json format
+json_str = casparser.read_cas_pdf("/path/to/cas/file.pdf", "password", output="json")
+
+# Get transactions data in csv string format
+csv_str = casparser.read_cas_pdf("/path/to/cas/file.pdf", "password", output="csv")
+
 ```
 
 ### Data structure
@@ -73,9 +80,8 @@ data = casparser.read_cas_pdf('/path/to/cas/pdf/file.pdf', 'password')
                             "units": "number",
                             "nav": "number",
                             "balance": "number",
-                            "is_dividend_payout": "boolean",
-                            "is_dividend_reinvestment": "boolean",
-                            "dividend_rate": null
+                            "type": "string",
+                            "dividend_rate": "number"
                         }
                     ]
                 }
@@ -84,8 +90,22 @@ data = casparser.read_cas_pdf('/path/to/cas/pdf/file.pdf', 'password')
     ]
 }
 ```
-
-
+Notes: 
+- Transaction `type` can be any value from the following
+  - `PURCHASE`
+  - `PURCHASE_SIP`
+  - `REDEMPTION`
+  - `SWITCH_IN`
+  - `SWITCH_IN_MERGER`
+  - `SWITCH_OUT`
+  - `SWITCH_OUT_MERGER`
+  - `DIVIDEND_PAYOUT`
+  - `DIVIDEND_REINVESTMENT`
+  - `TAX`
+  - `MISC`
+- `dividend_rate` is applicable only for `DIVIDEND_PAYOUT` and 
+  `DIVIDEND_REINVESTMENT` transactions.
+  
 ### CLI
 
 casparser also comes with a command-line interface that prints summary of parsed 
@@ -108,6 +128,10 @@ Usage: casparser [-o output_file.json] [-p password] [-s type] [-a] CAS_PDF_FILE
   -h, --help                      Show this message and exit.
 ``` 
 
+**Note:** `casparser cli` supports two special output file formats [-o _file.json_ / _file.csv_]
+1. `json` - complete parsed data is exported in json format (including investor info)
+2. `csv`  - transactions with AMC, Folio and Scheme info are exported into csv format. 
+
 #### Demo
 
 ![demo](https://raw.githubusercontent.com/codereverser/casparser/main/assets/demo.jpg)

README.md

@@ -1 +1 @@
-0.3.8
+0.3.9

casparser/VERSION.txt

@@ -1,5 +1,4 @@
 from decimal import Decimal
-import json
 import os
 import re
 import shutil
@@ -15,9 +14,8 @@
 from .__version__ import __version__
 
 from . import read_cas_pdf
-from .encoder import CASDataEncoder
 from .exceptions import ParserException
-from .parsers.utils import isclose
+from .parsers.utils import is_close, cas2json, cas2csv
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
@@ -94,7 +92,7 @@ def print_summary(data, tablefmt="fancy_grid", output_filename=None, include_zer
 
             # Check is calculated close (i.e. open + units from all transactions) is same as
             # reported close and also the scheme valuation = nav * calculated close.
-            if calc_close != scheme["close"] or not isclose(
+            if calc_close != scheme["close"] or not is_close(
                 valuation["nav"] * calc_close, valuation["value"], tol=2
             ):
                 err += 1
@@ -201,7 +199,7 @@ def cli(output, summary, password, include_all, force_pdfminer, filename):
     if output is not None:
         output_ext = os.path.splitext(output)[-1].lower()
 
-    if not (summary or output_ext == ".json"):
+    if not (summary or output_ext in (".csv", ".json")):
         summary = "fancy_grid"
 
     try:
@@ -214,11 +212,12 @@ def cli(output, summary, password, include_all, force_pdfminer, filename):
             data,
             tablefmt=summary,
             include_zero_folios=include_all,
-            output_filename=None if output_ext == ".json" else output,
+            output_filename=None if output_ext in (".csv", ".json") else output,
         )
-    if output_ext == ".json":
+    if output_ext in (".csv", ".json"):
+        conv_fn = cas2json if output_ext == ".json" else cas2csv
         with open(output, "w") as fp:
-            json.dump(data, fp, cls=CASDataEncoder, indent=2)
+            fp.write(conv_fn(data))
         click.echo("File saved : " + click.style(output, bold=True))
 
 

casparser/cli.py

@@ -1,9 +1,24 @@
-import enum
+from enum import Enum, IntEnum, auto
 
 
-class FileType(enum.IntEnum):
+class FileType(IntEnum):
     """Enum for CAS file source."""
 
     UNKNOWN = 0
     CAMS = 1
     KFINTECH = 2
+
+
+class TransactionType(Enum):
+    PURCHASE = auto()
+    PURCHASE_SIP = auto()
+    REDEMPTION = auto()
+    DIVIDEND_PAYOUT = auto()
+    DIVIDEND_REINVEST = auto()
+    SWITCH_IN = auto()
+    SWITCH_IN_MERGER = auto()
+    SWITCH_OUT = auto()
+    SWITCH_OUT_MERGER = auto()
+    TAX = auto()
+    MISC = auto()
+    UNKNOWN = auto()

casparser/enums.py

@@ -1,9 +1,8 @@
 import io
-import json
 from typing import Union
 
-from casparser.encoder import CASDataEncoder
 from casparser.process import process_cas_text
+from .utils import cas2json
 
 
 def read_cas_pdf(filename: Union[str, io.IOBase], password, output="dict", force_pdfminer=False):
@@ -35,4 +34,4 @@ def read_cas_pdf(filename: Union[str, io.IOBase], password, output="dict", force
     )
     if output == "dict":
         return processed_data
-    return json.dumps(processed_data, cls=CASDataEncoder)
+    return cas2json(processed_data)

casparser/parsers/__init__.py

@@ -8,7 +8,7 @@
 
 from casparser.enums import FileType
 from casparser.exceptions import CASParseError
-from .utils import isclose, InvestorInfo, PartialCASData
+from .utils import is_close, InvestorInfo, PartialCASData
 
 
 def extract_blocks(page_dict):
@@ -26,7 +26,7 @@ def extract_blocks(page_dict):
         y0, y1 = bbox[1], bbox[3]
         for line in sorted(block["lines"], key=lambda x: x["bbox"][1]):
             if len(items) > 0 and not (
-                isclose(y0, line["bbox"][1], tol=3) or isclose(y1, line["bbox"][3], tol=3)
+                is_close(y0, line["bbox"][1], tol=3) or is_close(y1, line["bbox"][3], tol=3)
             ):
                 full_text = "\t\t".join(
                     [x[0].strip() for x in sorted(items, key=lambda x: x[1][0]) if x[0].strip()]
@@ -119,7 +119,7 @@ def group_similar_rows(elements_list: List[Iterator[Any]]):
         y0, y1 = sorted_elements[0][1], sorted_elements[0][3]
         items = []
         for el in sorted_elements:
-            if len(items) > 0 and not (isclose(el[3], y1, tol=3) or isclose(el[1], y0, tol=3)):
+            if len(items) > 0 and not (is_close(el[3], y1, tol=3) or is_close(el[1], y0, tol=3)):
                 line = "\t\t".join(
                     [x[4].strip() for x in sorted(items, key=lambda x: x[0]) if x[4].strip()]
                 )
@@ -143,9 +143,7 @@ def cas_pdf_to_text(filename: Union[str, io.IOBase], password) -> PartialCASData
 
     if isinstance(filename, str):
         fp = open(filename, "rb")
-    elif isinstance(filename, io.IOBase):
-        fp = filename
-    elif hasattr(filename, "read"):  # compatibility for Django UploadedFile
+    elif hasattr(filename, "read") and hasattr(filename, "close"):  # file-like object
         fp = filename
     else:
         raise CASParseError("Invalid input. filename should be a string or a file like object")

casparser/parsers/mupdf.py

@@ -12,7 +12,7 @@
 
 from casparser.enums import FileType
 from casparser.exceptions import CASParseError
-from .utils import isclose, InvestorInfo, PartialCASData
+from .utils import is_close, InvestorInfo, PartialCASData
 
 
 def parse_investor_info(layout, width, height) -> InvestorInfo:
@@ -86,7 +86,7 @@ def group_similar_rows(elements_list: List[Iterator[LTTextBoxHorizontal]]):
         y0, y1 = sorted_elements[0].y0, sorted_elements[0].y1
         items = []
         for el in sorted_elements:
-            if len(items) > 0 and not (isclose(el.y1, y1, tol=3) or isclose(el.y0, y0, tol=3)):
+            if len(items) > 0 and not (is_close(el.y1, y1, tol=3) or is_close(el.y0, y0, tol=3)):
                 line = "\t\t".join(
                     [x.get_text().strip() for x in sorted(items, key=lambda x: x.x0)]
                 )
@@ -110,9 +110,7 @@ def cas_pdf_to_text(filename: Union[str, io.IOBase], password) -> PartialCASData
 
     if isinstance(filename, str):
         fp = open(filename, "rb")
-    elif isinstance(filename, io.IOBase):
-        fp = filename
-    elif hasattr(filename, "read"):  # compatibility for Django UploadedFile
+    elif hasattr(filename, "read") and hasattr(filename, "close"):  # file-like object
         fp = filename
     else:
         raise CASParseError("Invalid input. filename should be a string or a file like object")