Merge pull request #75 from DiamondLightSource/I04_1-415_backup_change

I04 1 415 backup change

Author: UrszulaNeuman

dls_barcode/config/barcode_config.py

@@ -22,8 +22,11 @@ def __init__(self, file, file_manager):
 
         if IS_BUNDLED:
             default_store = "./store/"
+            default_backup = "./backup/"
         else:
             default_store = "../store/"
+            default_backup = "../backup/"
+
 
         self.color_ok = add(ColorConfigItem, "Pin/Puck Read", Color.Green())
         self.color_accept = add(ColorConfigItem, "Puck Partially Read", Color.Yellow())
@@ -43,8 +46,9 @@ def __init__(self, file, file_manager):
         self.image_crop = add(BoolConfigItem, "Crop to Puck", default=True)
 
         self.store_directory = add(DirectoryConfigItem, "Store Directory", default=default_store)
-        self.store_capacity = add(IntConfigItem, "Results History Size", default=50)
-        self.backup_time = add(IntConfigItem, "Backup in Weeks", default=3)
+        self.backup = add(BoolConfigItem, "Backup before Delete", default=True)
+        self.backup_directory = add(DirectoryConfigItem, "Backup Directory", default=default_backup)
+
 
         self.console_frame = add(BoolConfigItem, "Print Frame Summary", default=False)
         self.slot_images = add(BoolConfigItem, "Save Debug Images", default=False)
@@ -60,6 +64,12 @@ def __init__(self, file, file_manager):
 
         self.initialize_from_file()
 
+    def get_store_directory(self):
+        return self.store_directory.value()
+
+    def get_backup_directory(self):
+        return self.backup_directory.value()
+
     def col_ok(self):
         return self.color_ok.value()
 

dls_barcode/config/barcode_config_dialog.py

@@ -55,8 +55,8 @@ def _init_ui(self):
 
         self.start_group("Store")
         self._add_control(StoreDirectoryConfigControl(cfg.store_directory))
-        add(cfg.backup_time)
-        add(cfg.store_capacity)
+        add(cfg.backup)
+        add(cfg.backup_directory)
 
         self.start_group("Debug")
         add(cfg.console_frame)

dls_barcode/data_store/backup.py

@@ -1,33 +1,13 @@
-import time
-
-from dls_barcode.data_store.comms_manager import CommsManager
-
-
 class Backup:
 
     """
     Backup class maintains the short time backaup of records which is kept in the same folder as the store files.
     """
 
-    WEEK_IN_SECONDS = 604800
-
-    def __init__(self, comms_man, backup_time):
+    def __init__(self, comms_man):
         self._comms = comms_man
-        self._backup_time = backup_time.value()
-        self._records = self._comms.load_records_from_file()
-
-    def _truncate_record_list(self):
-        self._records = list(filter(lambda x: not self._is_old(x), self._records))
 
     def backup_records(self, to_back):
-        self._records.extend(to_back)
-        self._truncate_record_list()
-        self._comms.to_csv_file(self._records)
+        self._comms.to_csv_file(to_back)
 
-    def _is_old(self, record):
-        tm = time.time()
-        record_time = record.timestamp
-        delta = tm - record_time
-        weeks = self._backup_time * self.WEEK_IN_SECONDS  # weeks in seconds
-        return delta > weeks
 

dls_barcode/data_store/store.py

@@ -1,6 +1,8 @@
 import uuid
-import os
+import time
 
+from dls_barcode.data_store.backup import Backup
+from dls_barcode.data_store.store_writer import StoreWriter
 from .record import Record
 
 
@@ -9,18 +11,12 @@ class Store:
     """ Maintains a list of records of previous barcodes scans. Any changes (additions
     or deletions) are automatically written to the backing file.
     """
-    MIN_STORE_CAPACITY = 2
 
-    def __init__(self, comms_manager, backup, store_capacity):
+    def __init__(self, store_writer, records):
         """ Initializes a new instance of Store.
         """
-        self._store_capacity = store_capacity
-        self._backup = backup
-        self._comms_manager = comms_manager
-        self._img_dir = self._comms_manager.make_img_dir()
-        self.records = self._comms_manager.load_records_from_file()
-        self._truncate_record_list()
-        self._sort_records()
+        self._store_writer = store_writer
+        self.records = records
 
     def size(self):
         """ Returns the number of records in the store
@@ -30,17 +26,17 @@ def size(self):
     def get_record(self, index):
         """ Get record by index where the 0th record is the most recent
         """
+        self._sort_records()
         return self.records[index] if self.records else None
 
     def _add_record(self, holder_barcode, plate, holder_img, pins_img):
         """ Add a new record to the store and save to the backing file.
         """
         merged_img = self._merge_holder_image_into_pins_image(holder_img, pins_img)
         guid = str(uuid.uuid4())
-        filename = os.path.abspath(os.path.join(self._img_dir, guid + '.png'))
-        merged_img.save_as(filename)
+        self._store_writer.to_image(merged_img, guid)
 
-        record = Record.from_plate(holder_barcode, plate, filename)
+        record = Record.from_plate(holder_barcode, plate, self._store_writer.get_img_path())
 
         self.records.append(record)
         self._process_change()
@@ -53,35 +49,30 @@ def merge_record(self, holder_barcode, plate, holder_img, pins_img):
 
         self._add_record(holder_barcode, plate, holder_img, pins_img)
 
+    def backup_records(self, directory):
+        ts = time.localtime()
+        file_name = time.strftime("%Y-%m-%d_%H-%M-%S", ts)
+        backup_writer = StoreWriter(directory, file_name)
+        backup = Backup(backup_writer)
+        self._sort_records()
+        backup.backup_records(self.records)
+
     def delete_records(self, records_to_delete):
         """ Remove all of the records in the supplied list from the store and
         save changes to the backing file.
         """
         for record in records_to_delete:
             self.records.remove(record)
-            self._comms_manager.remove_img_file(record)
+            self._store_writer.remove_img_file(record)
 
         self._process_change()
 
-    def backup_records(self, records_to_backup):
-        self._backup.backup_records(records_to_backup)
-
-    def _truncate_record_list(self):
-
-        actual_store_capacity = max(self._store_capacity.value(), self.MIN_STORE_CAPACITY)
-
-        if len(self.records) > actual_store_capacity:
-            to_delete = self.records[actual_store_capacity:]
-            self.backup_records(to_delete)
-            self.delete_records(to_delete)
-
     def _process_change(self):
         """ Sort the records and save to file.
         """
         self._sort_records()
-        self._truncate_record_list()
-        self._comms_manager.to_file(self.records)
-        self._comms_manager.to_csv_file(self.records)
+        self._store_writer.to_file(self.records)
+        self._store_writer.to_csv_file(self.records)
 
     def _sort_records(self):
         """ Sort the records in descending date order (most recent first).
@@ -97,5 +88,6 @@ def _merge_holder_image_into_pins_image(self, holder_img, pins_img):
         return merged_img
 
     def is_latest_holder_barcode(self, holder_barcode):
+        self._sort_records()
         latest_record = self.get_record(0)
         return latest_record is not None and holder_barcode == latest_record.holder_barcode

dls_barcode/data_store/store_loader.py

@@ -0,0 +1,39 @@
+import os
+
+from dls_barcode.data_store.record import Record
+from dls_util.file import FileManager
+
+
+class StoreLoader:
+
+    def __init__(self, directory, file_name, file_manager=FileManager()):
+        self._directory = directory
+        self._file_name = file_name
+        self._file_manager= file_manager
+        self._path = None
+        self._records = []
+
+    def load_records_from_file(self):
+        """ Clear the current record store and load a new set of records from the specified file. """
+        self._build_file_path()
+        if not self._check_if_file():
+            return self._records
+        self._build_records_from_lines(self._read_lines())
+        return self._records
+
+    def _build_file_path(self):
+        self._path =  os.path.join(self._directory, self._file_name + '.txt')
+
+    def _check_if_file(self):
+        return self._file_manager.is_file(self._path)
+
+    def _read_lines(self):
+        return self._file_manager.read_lines(self._path)
+
+    def _build_records_from_lines(self, lines):
+        for line in lines:
+            try:
+                record = Record.from_string(line)
+                self._records.append(record)
+            except Exception:
+                print("Failed to parse store Record: {}".format(line))

dls_barcode/data_store/store_manager.py

@@ -4,51 +4,44 @@
 from dls_util.file import FileManager
 
 
-class CommsManager:
-    """ Maintains communication between store/backup and the file manager.
+class StoreWriter:
+    """ Maintains writing records to file and saving png images in a sub-folder
     """
 
-    def __init__(self, directory, file_name):
-        self._file_manager = FileManager()
-        self._directory = directory.value()
+    def __init__(self, directory, file_name, file_manager=FileManager()):
+        self._file_manager = file_manager
+        self._directory = directory
         self._file_name = file_name
-
-    def load_records_from_file(self):
-        """ Clear the current record store and load a new set of records from the specified file. """
-        file = os.path.join(self._directory, self._file_name + '.txt')
-        records = []
-
-        if not self._file_manager.is_file(file):
-            return records
-
-        lines = self._file_manager.read_lines(file)
-        for line in lines:
-            try:
-                record = Record.from_string(line)
-                records.append(record)
-            except Exception:
-                print("Failed to parse store Record: {}".format(line))
-
-        return records
+        self._image_path = None
 
     def to_file(self, records):
         """ Save the contents of the store to the backing file
         """
+        self._file_manager.make_dir_when_no_dir(self._directory)
         file = os.path.join(self._directory, self._file_name + '.txt')
         record_lines = [rec.to_string() + "\n" for rec in records]
         self._file_manager.write_lines(file, record_lines)
 
     def to_csv_file(self, records):
         """ Save the contents of the store to the backing csv file
         """
+        self._file_manager.make_dir_when_no_dir(self._directory)
         csv_file = os.path.join(self._directory, self._file_name + ".csv")
         record_lines = [rec.to_csv_string() + "\n" for rec in records]
         self._file_manager.write_lines(csv_file, record_lines)
 
-    def make_img_dir(self):
+    def to_image(self, image, name):
+        dr = self._make_img_dir()
+        self._image_path = os.path.abspath(os.path.join(dr, name + '.png'))
+        image.save_as(self._image_path)
+
+    def get_img_path(self):
+        return self._image_path
+
+    def _make_img_dir(self):
+        self._file_manager.make_dir_when_no_dir(self._directory)
         img_dir = os.path.join(self._directory, "img_dir")
-        if not self._file_manager.is_dir(img_dir):
-            self._file_manager.make_dir(img_dir)
+        self._file_manager.make_dir_when_no_dir(img_dir)
         return img_dir
 
     def remove_img_file(self, record):

dls_barcode/data_store/comms_manager.py dls_barcode/data_store/store_writer.pydls_barcode/data_store/comms_manager.py renamed to dls_barcode/data_store/store_writer.py

@@ -58,11 +58,14 @@ def interpret_bytes(data_bytes):
                 #log.error(NotImplementedError("Datamatrix Extended Channel Interpretation code not implemented"))
                 raise NotImplementedError("Datamatrix Extended Channel Interpretation code not implemented")
 
-            elif 242 <= byte < 256 or byte == 0:  # Unused parts of message space.
+            elif 242 <= byte < 256:  # Unused parts of message space.
                 #log = log = logging.getLogger(".".join([__name__]))
                 error = ValueError("Code {} is not used in Datamatrix specification".format(byte))
                 #.error(error)
                 raise error
+            elif byte == 0:
+                error = ValueError("Code {} is not used in Datamatrix specification".format(byte))
+                print(error)
 
         return ''.join(m for m in message)
 

dls_barcode/datamatrix/read/interpret.py

@@ -5,8 +5,8 @@
 from PyQt5.QtWidgets import QGroupBox, QVBoxLayout, QHBoxLayout, QTableWidget, QMessageBox
 
 from dls_barcode.data_store import Store
-from dls_barcode.data_store.store_manager import StoreManager
-from dls_util.file import FileManager
+from dls_barcode.data_store.store_loader import StoreLoader
+from dls_barcode.data_store.store_writer import StoreWriter
 
 # todo: allow delete key to be used for deletion
 # todo: allow record selection with arrow keys
@@ -23,7 +23,10 @@ def __init__(self, barcode_table, image_frame, options):
         super(ScanRecordTable, self).__init__()
 
         # Read the store from file
-        self._store = StoreManager(options.store_directory, options.store_capacity, options.backup_time).create_store()
+        store_writer = StoreWriter(options.get_store_directory(), "store")
+        store_loader = StoreLoader(options.get_store_directory(), "store")
+
+        self._store = Store(store_writer, store_loader.load_records_from_file())
         self._options = options
 
         self._barcodeTable = barcode_table
@@ -136,7 +139,8 @@ def _delete_selected_records(self):
                 record = self._store.get_record(index)
                 records_to_delete.append(record)
 
-            self._store.backup_records(records_to_delete)
+            if self._options.backup.value():
+                self._store.backup_records(self._options.get_backup_directory())
             self._store.delete_records(records_to_delete)
 
             self._load_store_records()

dls_barcode/gui/record_table.py

@@ -29,4 +29,8 @@ def make_dir(self, path):
         os.makedirs(path)
 
     def remove(self, path):
-        os.remove(path)
+        os.remove(path)
+
+    def make_dir_when_no_dir(self, dr):
+        if not self.is_dir(dr):
+            self.make_dir(dr)