Use chardet to preserve file encoding

Author: MHendricks

preditor/gui/workbox_mixin.py

@@ -1,9 +1,11 @@
 from __future__ import absolute_import, print_function
 
+import io
 import os
 import tempfile
 import textwrap
 
+import chardet
 from Qt.QtCore import Qt
 from Qt.QtWidgets import QStackedWidget
 
@@ -317,14 +319,37 @@ def __remove_tempfile__(self):
             os.remove(tempfile)
 
     @classmethod
-    def __open_file__(cls, filename):
-        with open(filename) as fle:
-            return fle.read()
-        return ""
+    def __open_file__(cls, filename, strict=True):
+        """Open a file and try to detect the text encoding it was saved as.
+
+        Returns:
+            encoding(str): The detected encoding, Defaults to "utf-8" if unable
+                to detect encoding.
+            text(str): The contents of the file decoded to a str.
+        """
+        with open(filename, "rb") as f:
+            text_bytes = f.read()
+
+        # Open file, detect source encoding and convert to utf-8
+        encoding = chardet.detect(text_bytes)['encoding'] or 'utf-8'
+        try:
+            text = text_bytes.decode(encoding)
+        except UnicodeDecodeError as e:
+            if strict:
+                raise UnicodeDecodeError(  # noqa: B904
+                    e.encoding,
+                    e.object,
+                    e.start,
+                    e.end,
+                    f"{e.reason}, Filename: {filename}",
+                )
+            encoding = 'utf-8'
+            text = text_bytes.decode(encoding, errors="ignore")
+        return encoding, text
 
     @classmethod
-    def __write_file__(cls, filename, txt):
-        with open(filename, 'w') as fle:
+    def __write_file__(cls, filename, txt, encoding=None):
+        with io.open(filename, 'w', newline='\n', encoding=encoding) as fle:
             fle.write(txt)
 
     def __show__(self):
@@ -335,7 +360,7 @@ def __show__(self):
         if self._filename_pref:
             self.__load__(self._filename_pref)
         elif self._tempfile:
-            txt = self.__open_file__(self.__tempfile__())
+            _, txt = self.__open_file__(self.__tempfile__(), strict=False)
             self.__set_text__(txt)
 
     def process_shortcut(self, event, run=True):

preditor/gui/workbox_text_edit.py

@@ -27,6 +27,7 @@ def __init__(
     ):
         super(WorkboxTextEdit, self).__init__(parent=parent, core_name=core_name)
         self._filename = None
+        self._encoding = None
         self.__set_console__(console)
         highlight = CodeHighlighter(self)
         highlight.setLanguage('Python')
@@ -79,7 +80,8 @@ def __set_indentations_use_tabs__(self, state):
 
     def __load__(self, filename):
         self._filename = filename
-        txt = self.__open_file__(self._filename)
+        enc, txt = self.__open_file__(self._filename)
+        self._encoding = enc
         self.__set_text__(txt)
 
     def __margins_font__(self):

preditor/gui/workboxwidget.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import, print_function
 
-import io
 import re
 import time
 
@@ -197,10 +196,10 @@ def __set_text__(self, txt):
         self.setText(txt)
 
     @classmethod
-    def __write_file__(cls, filename, txt):
-        with io.open(filename, 'w', newline='\n') as fle:
-            # Save unix newlines for simplicity
-            fle.write(cls.__unix_end_lines__(txt))
+    def __write_file__(cls, filename, txt, encoding=None):
+        # Save unix newlines for simplicity
+        txt = cls.__unix_end_lines__(txt)
+        super(WorkboxWidget, cls).__write_file__(filename, txt, encoding=encoding)
 
     def keyPressEvent(self, event):
         """Check for certain keyboard shortcuts, and handle them as needed,

preditor/scintilla/documenteditor.py

@@ -20,9 +20,8 @@
 from functools import partial
 
 import six
-from PyQt5.QtCore import QTextCodec
 from Qt import QtCompat
-from Qt.QtCore import Property, QFile, QPoint, Qt, Signal
+from Qt.QtCore import Property, QPoint, Qt, Signal
 from Qt.QtGui import QColor, QFont, QFontMetrics, QIcon
 from Qt.QtWidgets import (
     QAction,
@@ -37,6 +36,7 @@
 from ..delayable_engine import DelayableEngine
 from ..enum import Enum, EnumGroup
 from ..gui import QtPropertyInit
+from ..gui.workbox_mixin import WorkboxMixin
 from . import QsciScintilla, lang
 
 logger = logging.getLogger(__name__)
@@ -90,7 +90,7 @@ def __init__(self, parent, filename='', lineno=0, delayable_engine='default'):
         self.additionalFilenames = []
         self._language = ''
         self._lastSearch = ''
-        self._textCodec = None
+        self._encoding = 'utf-8'
         self._fileMonitoringActive = False
         self._marginsFont = self._defaultFont
         self._lastSearchDirection = SearchDirection.First
@@ -657,14 +657,8 @@ def lineMarginWidth(self):
     def load(self, filename):
         filename = str(filename)
         if filename and os.path.exists(filename):
-            f = QFile(filename)
-            f.open(QFile.ReadOnly)
-            text = f.readAll()
-            self._textCodec = QTextCodec.codecForUtfText(
-                text, QTextCodec.codecForName('UTF-8')
-            )
-            self.setText(self._textCodec.toUnicode(text))
-            f.close()
+            self._encoding, text = WorkboxMixin.__open_file__(filename)
+            self.setText(text)
             self.updateFilename(filename)
             self.enableFileWatching(True)
             self.setEolMode(self.detectEndLine(self.text()))
@@ -1091,25 +1085,21 @@ def saveAs(self, filename='', setFilename=True):
         if filename:
             self._saveTimer = time.time()
             # save the file to disk
-            f = QFile(filename)
-            f.open(QFile.WriteOnly)
-            # make sure the file is writeable
-            if f.error() != QFile.NoError:
-                logger.debug('An error occured while saving')
+            try:
+                txt = self.text()
+                WorkboxMixin.__write_file__(filename, txt, encoding=self._encoding)
+                with open(filename, "w", encoding=self._encoding) as f:
+                    f.write(self.text())
+            except PermissionError as error:
+                logger.debug('An error occurred while saving')
                 QMessageBox.question(
                     self.window(),
                     'Error saving file...',
-                    'There was a error saving the file. Error Code: %i' % f.error(),
+                    'There was a error saving the file. Error: {}'.format(error),
                     QMessageBox.StandardButton.Ok,
                 )
-                f.close()
                 return False
-            # Attempt to save the file using the same codec that it used to display it
-            if self._textCodec:
-                f.write(self._textCodec.fromUnicode(self.text()))
-            else:
-                self.write(f)
-            f.close()
+
             # notify that the document was saved
             self.documentSaved.emit(self, filename)
 
@@ -1710,10 +1700,8 @@ def updateSelectionInfo(self):
                     epos=epos,
                     lineCount=eline - sline + 1,
                 )
-            if self._textCodec and self._textCodec.name() != 'System':
-                text = 'Encoding: {enc} {text}'.format(
-                    enc=self._textCodec.name(), text=text
-                )
+            if self._encoding:
+                text = 'Encoding: {enc} {text}'.format(enc=self._encoding, text=text)
             window.uiCursorInfoLBL.setText(text)
 
     def setAutoReloadOnChange(self, state):

setup.cfg

@@ -25,6 +25,7 @@ platform = any
 packages = find:
 install_requires =
     Qt.py
+    chardet
     configparser>=4.0.2
     future>=0.18.2
     python-redmine>=2.1.1