API ready

Author: Sven Siegmund

README.md

@@ -44,18 +44,22 @@ Created directory C:\Users\nagidal\rtfparse\html
 
 `rtfparse` also creates the folder `.rtfparse` (beginning with a dot) in your home directory where it saves its default configuration and its log files.
 
-# Usage
+# Usage From Command Line
 
 Use the `rtfparse` executable from the command line. For example if you want to de-encapsulate the HTML from an RTF file, do it like this:
 
     rtfparse -f "path/to/rtf_file.rtf" -d "path/to/de_encapsulated.html"
 
 Or you can de-encapsulate the HTML from an MS Outlook message, thanks to [extract_msg](https://github.com/TeamMsgExtractor/msg-extractor) and [compressed_rtf](https://github.com/delimitry/compressed_rtf):
 
-    rtfparse -m "path/to/email.msg" -d "path/to/de_encapsulated.html"
+    rtfparse -m "path/to/email.msg" -d
 
 Command reference is in `rtfparse --help`.
 
+# Usage in python module
+
+See 'minimal.py' for an example.
+
 # RTF Specification Links
 
 If you find a working official Microsoft link to the RTF specification and add it here, you'll be remembered fondly.

src/rtfparse/entities.py

@@ -8,7 +8,6 @@
 from rtfparse import re_patterns
 from rtfparse import utils
 from rtfparse import errors
-from rtfparse import config_loader
 from rtfparse.enums import Bytestring_Type
 
 
@@ -53,8 +52,10 @@ def probe(cls, pattern: re_patterns.Bytes_Regex, file: io.BufferedReader) -> Byt
                 file.seek(original_position)
                 logger.debug(f"Probe moved to position {file.tell()}")
                 if not probed:
-                    logger.warning(f"Reached unexpected end of file.")
-                    raise errors.UnexpectedEndOfFileError(f"at position {file.tell()}")
+                    logger.debug(f"Reached unexpected end of file.")
+                    result = Bytestring_Type.GROUP_END
+                    break
+                    # raise errors.UnexpectedEndOfFileError(f"at position {file.tell()}")
                 continue
             break
         logger.debug(f"Probe {result = }")
@@ -63,9 +64,9 @@ def probe(cls, pattern: re_patterns.Bytes_Regex, file: io.BufferedReader) -> Byt
 
 
 class Control_Word(Entity):
-    def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> None:
+    def __init__(self, encoding: str, file: io.BufferedReader) -> None:
         super().__init__()
-        self.config = config
+        self.encoding = encoding
         logger.debug(f"Reading Control Word at file position {file.tell()}")
         self.control_name = "missing"
         self.parameter = ""
@@ -74,17 +75,17 @@ def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> Non
         logger.debug(f"Starting at file position {self.start_position}")
         probe = file.read(CONTROL_WORD)
         if (match := re_patterns.control_word.match(probe)):
-            self.control_name = match.group("control_name").decode(self.config.default_encoding)
+            self.control_name = match.group("control_name").decode(self.encoding)
             logger.debug(f"Preliminary {self.control_name = }")
             parameter = match.group("parameter")
             if parameter is not None:
-                self.parameter = int(parameter.decode(self.config.default_encoding))
+                self.parameter = int(parameter.decode(self.encoding))
                 logger.debug(f"{self.parameter = }")
                 self.control_name = self.control_name.removesuffix(str(self.parameter))
                 logger.debug(f"Final {self.control_name = }")
             target_position = self.start_position + match.span()[1]
             if match.group("other"):
-                logger.debug(f"Delimiter is {match.group('other').decode(self.config.default_encoding)}, len: {len(match.group('delimiter'))}")
+                logger.debug(f"Delimiter is {match.group('other').decode(self.encoding)}, len: {len(match.group('delimiter'))}")
                 target_position -= len(match.group("delimiter"))
             file.seek(target_position)
             # handle \binN:
@@ -98,16 +99,16 @@ def __repr__(self) -> str:
 
 
 class Control_Symbol(Entity):
-    def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> None:
+    def __init__(self, encoding: str, file: io.BufferedReader) -> None:
         super().__init__()
-        self.config = config
+        self.encoding = encoding
         self.start_position = file.tell()
         logger.debug(f"Reading Symbol at file position {self.start_position}")
         self.char = ""
         self.text = chr(file.read(SYMBOL)[-1])
         if self.text == "'":
-            self.char = file.read(SYMBOL).decode(self.config.default_encoding)
-            self.text = bytes((int(self.char, base=16), )).decode(self.config.default_encoding)
+            self.char = file.read(SYMBOL).decode(self.encoding)
+            self.text = bytes((int(self.char, base=16), )).decode(self.encoding)
             logger.debug(f"Encountered escaped ANSI character, read two more bytes: {self.char}, character: {self.text}")
             if self.text in "\\{}":
                 file.seek(file.tell() - SYMBOL)
@@ -116,9 +117,9 @@ def __repr__(self) -> str:
 
 
 class Plain_Text(Entity):
-    def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> None:
+    def __init__(self, encoding: str, file: io.BufferedReader) -> None:
         super().__init__()
-        self.config = config
+        self.encoding = encoding
         self.text = ""
         logger.debug(f"Constructing Plain_Text")
         while True:
@@ -128,7 +129,7 @@ def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> Non
             # see if we have read all the plain text there is:
             if (match := re_patterns.plain_text.match(read)):
                 logger.debug(f"This matches the plain text pattern")
-                _text = match.group("text").decode(self.config.default_encoding)
+                _text = match.group("text").decode(self.encoding)
                 logger.debug(f"{_text = }")
                 self.text = "".join((self.text, _text))
                 logger.debug(f"{self.text = }")
@@ -146,10 +147,10 @@ def __repr__(self) -> str:
 
 
 class Group(Entity):
-    def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> None:
+    def __init__(self, encoding: str, file: io.BufferedReader) -> None:
         super().__init__()
         logger.debug(f"Group.__init__")
-        self.config = config
+        self.encoding = encoding
         self.known = False
         self.name = "unknown"
         self.ignorable = False
@@ -164,24 +165,24 @@ def __init__(self, config: config_loader.Config, file: io.BufferedReader) -> Non
             self.known = bool(match.group("group_start"))
             self.ignorable = bool(match.group("ignorable"))
             if not self.ignorable:
-                file.seek(-IGNORABLE, io.SEEK_CUR)
+                file.seek(self.start_position + GROUP_START - IGNORABLE)
                 logger.debug(f"Returned to position {file.tell()}")
         else:
             logger.warning(utils.warn(f"Expected a group but found no group start. Creating unknown group"))
             file.seek(self.start_position)
         while True:
             probed = self.probe(re_patterns.probe, file)
             if probed is Bytestring_Type.CONTROL_WORD:
-                self.structure.append(Control_Word(self.config, file))
+                self.structure.append(Control_Word(self.encoding, file))
             elif probed is Bytestring_Type.GROUP_END:
                 file.read(GROUP_END)
                 break
             elif probed is Bytestring_Type.GROUP_START:
-                self.structure.append(Group(self.config, file))
+                self.structure.append(Group(self.encoding, file))
             elif probed is Bytestring_Type.CONTROL_SYMBOL:
-                self.structure.append(Control_Symbol(self.config, file))
+                self.structure.append(Control_Symbol(self.encoding, file))
             else:
-                self.structure.append(Plain_Text(self.config, file))
+                self.structure.append(Plain_Text(self.encoding, file))
         # name the group like its first Control Word
         # this way the renderer will be able to ignore entire groups based on their first control word
         try:

src/rtfparse/entry.py

@@ -19,7 +19,7 @@
 from rtfparse import config_loader
 from rtfparse import version
 from rtfparse.parser import Rtf_Parser
-from rtfparse.renderers import encapsulated_html
+from rtfparse.renderers import de_encapsulate_html
 
 
 # Setup logging
@@ -51,24 +51,26 @@ def argument_parser() -> argparse.ArgumentParser:
     parser.add_argument("--autoconfig", action="store_true", help="Configure rtfparse automatically").completer = EC
     parser.add_argument("-f", "--file", action="store", metavar="PATH", type=pathlib.Path, help="path to the rtf file").completer = EC
     parser.add_argument("-m", "--msg", action="store", metavar="PATH", type=pathlib.Path, help="Parse RTF from MS Outlook's .msg file").completer = EC
-    parser.add_argument("-d", "--de-encapsulate-html", action="store", metavar="PATH", type=pathlib.Path, help="De-encapsulate HTML from RTF").completer = EC
+    parser.add_argument("-d", "--de-encapsulate-html", action="store_true", help="De-encapsulate HTML from RTF").completer = EC
     return parser
 
 
 def de_encapsulate(rp: Rtf_Parser, target_file: pathlib.Path) -> None:
-    renderer = encapsulated_html.Encapsulated_HTML()
-    with open(target_file, mode="w", encoding="utf-8") as htmlfile:
+    renderer = de_encapsulate_html.De_encapsulate_HTML()
+    with open(target_file, mode="w", encoding="cp1252") as htmlfile:
         logger.info(f"Rendering the encapsulated HTML")
         renderer.render(rp.parsed, htmlfile)
         logger.info(f"Encapsulated HTML rendered")
 
 
 def run(config: config_loader.Config) -> None:
-    rp = Rtf_Parser()
     if config.cli_args.file and config.cli_args.file.exists():
+        file_name = config.cli_args.file.name
         with open(config.cli_args.file, mode="rb") as rtf_file:
-            rp.parse_file(config, rtf_file)
+            rp = Rtf_Parser(rtf_file=rtf_file)
+            rp.parse_file()
     elif config.cli_args.msg:
+        file_name = config.cli_args.msg.name
         msg = em.openMsg(f"{config.cli_args.msg}")
         for attachment in msg.attachments:
             with open(config.html / f"{attachment.longFilename}", mode="wb") as att_file:
@@ -77,9 +79,10 @@ def run(config: config_loader.Config) -> None:
         with open((config.email_rtf / config.cli_args.msg.name).with_suffix(".rtf"), mode="wb") as email_rtf:
             email_rtf.write(decompressed_rtf)
         with io.BytesIO(decompressed_rtf) as rtf_file:
-            rp.parse_file(config, rtf_file)
+            rp = Rtf_Parser(rtf_file=rtf_file)
+            rp.parse_file()
     if config.cli_args.de_encapsulate_html:
-        de_encapsulate(rp, config.cli_args.de_encapsulate_html)
+        de_encapsulate(rp, (config.html / file_name).with_suffix(".html"))
 
 
 def cli_start(version) -> None:

src/rtfparse/minimal.py

@@ -2,16 +2,17 @@
 
 
 import pathlib
-from rtfparse.parsers import Rtf_Parser
-from rtfparse.renderers import encapsulated_html
+from rtfparse.parser import Rtf_Parser
+from rtfparse.renderers import de_encapsulate_html
 
 
-source_file = pathlib.Path(r"D:\trace\email\test_mail_sw_release.rtf")
-target_file = pathlib.Path(r"D:\trace\email\extracted_with_rtfparse.html")
+source_path = pathlib.Path(r"D:\trace\email\test_mail_sw_release.rtf")
+target_path = pathlib.Path(r"D:\trace\email\extracted_with_rtfparse.html")
 
 
-parser = Rtf_Parser(rtf_file=source_file)
+parser = Rtf_Parser(rtf_path=source_path)
 parsed = parser.parse_file()
 
-renderer = encapsulated_html.Encapsulated_HTML()
-renderer.render(rtf_structure=parsed, target_file=target_file)
+renderer = de_encapsulate_html.De_encapsulate_HTML()
+with open(target_path, mode="w", encoding="utf-8") as html_file:
+    renderer.render(parsed, html_file)

src/rtfparse/parser.py

@@ -4,12 +4,14 @@
 import io
 import re
 import logging
+import pathlib
 # Own modules
 from rtfparse import re_patterns
 from rtfparse import entities
 from rtfparse import errors
 from rtfparse import utils
 # Typing
+from typing import Optional
 from typing import Union
 from rtfparse import config_loader
 
@@ -19,19 +21,65 @@
 
 
 class Rtf_Parser:
-    def __init__(self) -> None:
-        self.parsed = None
-    def parse_file(self, config: config_loader.Config, file: Union[io.BufferedReader, io.BytesIO]) -> None:
+    def __init__(self,
+                rtf_path: Optional[pathlib.Path]=None,
+                rtf_file: Optional[Union[io.BufferedReader, io.BytesIO]]=None,
+            ) -> None:
+        self.rtf_path = rtf_path
+        self.rtf_file = rtf_file
+        if not (self.rtf_path or self.rtf_file):
+            raise ValueError("Need `rtf_path` or `rtf_file` argument")
+        self.ENCODING_PROBE = 48 # look for encoding information in the first 48 bytes of the file
+    def read_encoding(self, file: Union[io.BufferedReader, io.BytesIO]) -> str:
+        probed = file.read(self.ENCODING_PROBE)
+        group = entities.Group("cp1252", io.BytesIO(probed))
+        recognized_encodings = (
+                    "ansi",
+                    "ansicpg",
+                    "mac",
+                    "pc",
+                    "pca",
+                )
+        names = tuple(filter(lambda item: isinstance(item, entities.Control_Word) and item.control_name in recognized_encodings, group.structure))
+        # Check if the ANSI code page is set:
+        cp = None
+        for item in names:
+            # if any item is a Control_Word which has a parameter, we assume that this is the parameter of \ansicpg, and that corresponds to the codepage we are looking for
+            if item.parameter:
+                param = item.parameter
+        if not param:
+            if names[0].control_name == "ansi":
+                encoding = "ansi"
+            elif names[0].control_name == "mac":
+                encoding = "mac_roman"
+            elif names[0].control_name == "pc":
+                encoding = "cp437"
+            elif names[0].control_name == "pca":
+                encoding = "cp850"
+        else:
+            encoding = f"cp{param}"
+        file.seek(0)
+        return encoding
+    def parse_file(self) -> entities.Group:
+        if self.rtf_path is not None:
+            file = open(self.rtf_path, mode="rb")
+        elif self.rtf_file is not None:
+            file = self.rtf_file
+        else:
+            file = io.BytesIO(b"")
         parsed_object = utils.what_is_being_parsed(file)
         logger.info(f"Parsing the structure of {parsed_object}")
         try:
-            self.parsed = entities.Group(config, file)
-        except errors.UnexpectedEndOfFileError as err:
-            logger.error(f"{err}")
+            encoding = self.read_encoding(file)
+            self.parsed = entities.Group(encoding, file)
         except Exception as err:
             logger.exception(err)
         finally:
+            if self.rtf_path is not None:
+                logger.debug(f"Closing {parsed_object}")
+                file.close()
             logger.info(f"Structure of {parsed_object} parsed")
+            return self.parsed
 
 
 if __name__ == "__main__":

…/rtfparse/renderers/encapsulated_html.py …tfparse/renderers/de_encapsulate_html.pysrc/rtfparse/renderers/encapsulated_html.py renamed to src/rtfparse/renderers/de_encapsulate_html.py src/rtfparse/renderers/encapsulated_html.py renamed to src/rtfparse/renderers/de_encapsulate_html.py

@@ -13,7 +13,7 @@
 logger = logging.getLogger(__name__)
 
 
-class Encapsulated_HTML(Renderer):
+class De_encapsulate_HTML(Renderer):
     def __init__(self, ) -> None:
         super().__init__()
         self.ignore_rtf = False
@@ -27,6 +27,8 @@ def __init__(self, ) -> None:
         self.ignore_groups = (
                 "fonttbl",
                 "colortbl",
+                "generator",
+                "formatConverter",
             )
     def ignore_rtf_toggle(self, cw: entities.Control_Word) -> str:
         if cw.parameter == "" or cw.parameter == 1: