summary parser: fix bugs with pdfminer backend and update test cases

Author: codereverser

casparser/process/cas_summary.py

@@ -43,11 +43,11 @@ def process_summary_text(text):
     for line in lines:
         if len(folios) > 0 and re.search("Total", line, re.I):
             break
-        scheme_tail = ""
-        if m := re.search(SCHEME_TAIL_RE, line, re.DOTALL | re.MULTILINE):
-            scheme_tail = m.group(1).strip()
-            line = line.replace(scheme_tail, "")
-            scheme_tail = re.sub(r"\s+", " ", scheme_tail).strip()
+        scheme_tails = []
+        if m := re.findall(SCHEME_TAIL_RE, line):
+            for txt in m:
+                line = line.replace(txt, "")
+                scheme_tails.append(re.sub(r"\s+", " ", txt).strip())
         if m := re.search(SUMMARY_ROW_RE, line, re.DOTALL | re.MULTILINE | re.I):
             folio = m.group("folio").strip()
             if current_folio is None or current_folio != folio:
@@ -61,8 +61,8 @@ def process_summary_text(text):
                     schemes=[],
                 )
             scheme = m.group("name")
-            if scheme_tail != "":
-                scheme = " ".join([scheme, scheme_tail])
+            if len(scheme_tails) > 0:
+                scheme = " ".join([scheme, *scheme_tails])
             scheme = re.sub(r"\(formerly.+?\)", "", scheme, flags=re.I | re.DOTALL).strip()
             rta = m.group("rta").strip()
             rta_code = m.group("code").strip()

casparser/process/regex.py

@@ -44,4 +44,4 @@
 TRANSACTION_RE3 = rf"{date_re}\t\t([^0-9].*)\t\t{amt_re}(?:\t\t{amt_re}\t\t{amt_re}\t\t{amt_re})*"
 DESCRIPTION_TAIL_RE = r"(\n.+?)(\t\t|$)"
 DIVIDEND_RE = r"(?:div\.|dividend|idcw).+?(reinvest)*.*?@\s*Rs\.\s*([\d\.]+)(?:\s+per\s+unit)?"
-SCHEME_TAIL_RE = r"(\n.+)$"
+SCHEME_TAIL_RE = r"(\n.+?)(?:\t\t|$)"

tests/base.py

@@ -56,20 +56,19 @@ def test_output_json(self):
 
     def test_read_summary(self):
         summary_files = (
-            (self.cams_summary_file_name, self.cams_password, 4),
-            (
-                self.kfintech_summary_file_name,
-                self.kfintech_password,
-                9,
-            ),
+            (self.cams_summary_file_name, self.cams_password, 4, 6),
+            (self.kfintech_summary_file_name, self.kfintech_password, 9, 13),
         )
-        for filename, password, num_folios in summary_files:
+        for filename, password, num_folios, num_schemes in summary_files:
             data = self.read_pdf(filename, password)
             assert len(data.folios) == num_folios
+            schemes_found = 0
             for folio in data.folios:
+                schemes_found += len(folio.schemes)
                 for scheme in folio.schemes:
                     assert scheme.isin is not None
                     assert scheme.amfi is not None
+            assert schemes_found == num_schemes
             assert data.investor_info.mobile not in (None, "")
             assert data.cas_type == CASFileType.SUMMARY.value