1.1.0: Added autosaving

Enabling autosaving to disk in a user-defined time interval.
The file content can be read using a new function "recover".
With each autosave, an automatic backup file (filename + ".tmp") is
created. that is used during recovery if the main file is broken.

Author: 2xB

example.py

@@ -1,5 +1,7 @@
 import faultguard
 import numpy as np
+import os
+import time
 
 def launch(faultguard_data, args):
     """
@@ -16,6 +18,7 @@ def launch(faultguard_data, args):
 
     # Some dummy important data manipulation
     for i in range(10):
+        time.sleep(1)
         important_data_1[i%3] = i
         important_data_2 += str(i)
         print("important_data_1:", important_data_1)
@@ -29,7 +32,7 @@ def launch(faultguard_data, args):
         if i == 7:
             import ctypes
             ctypes.string_at(0)
-            
+
 def rescue(faultguard_data, exit_code, args):
     """
     Demo rescue handler
@@ -40,6 +43,9 @@ def rescue(faultguard_data, exit_code, args):
     """
     print("Fault occured. Exit code: {}. Rescued data:".format(exit_code))
 
+    recover(faultguard_data)
+
+def recover(faultguard_data):
     # Check if fault occurs before data was initialized
     if "important_data_1" not in faultguard_data or "important_data_2" not in faultguard_data:
         return
@@ -51,9 +57,17 @@ def rescue(faultguard_data, exit_code, args):
     # You might need to assign the class here by important_data_1.__class__ = ...
     print("important_data_1:", important_data_1)
     print("important_data_2:", important_data_2)
-    
-def main():
-    faultguard.start(launch, rescue, args=("Hello", "World"))
+
+def main(use_autosave=True):
+    if use_autosave:
+        if os.path.isfile("test.tmp.xz"):
+            print("Autosave exists:")
+            faultguard.recover(recover, "test.tmp.xz")
+            os.remove("test.tmp.xz")
+        
+        faultguard.start(launch, rescue, args=("Hello", "World"), autosave_interval=3, autosave_file="test.tmp.xz")
+    else:
+        faultguard.start(launch, rescue, args=("Hello", "World"))
 
 if __name__ == "__main__":
     main()

faultguard.py

@@ -32,25 +32,81 @@ def wrapped_launch(launch, managed_dict, signal_handlers, args):
     # Attach signal handlers
     for sig in signal_handlers:
         signal.signal(sig, signal_handlers[sig])
-    
+
     faultguard_data = FaultguardDict(managed_dict)
     if args is None:
         launch(faultguard_data)
     else:
         launch(faultguard_data, args)
 
-def start(launch, rescue, args = None):
+def recover(rescue, autosave_file=None):
+    """
+    Load the given faultguard data dictionary from an autosave file and pass it to a rescue function.
+    
+    :param rescue: The method to call with the recovered faultguard data dictionary.
+    :param autosave_file: The file to recover the data from.
+    :returns: Whether the main (0) or the backup file (1) was used for recovery
+    """
+    # Compression library
+    import lzma
+    import os
+
+    success = True
+    try:
+        with lzma.open(autosave_file, "r") as f:
+            faultguard_data = FaultguardDict(pickle.load(f))
+    except Exception as e:
+        print("The following issue occured during recovery:", e)
+        success = False
+    
+    if success:
+        rescue(faultguard_data)
+        return 0
+    
+    if not os.path.isfile(autosave_file + ".tmp"):
+        raise RuntimeError("Recovery unsuccessful.")
+    
+    print("Switching to try recovery of backup file")
+
+    with lzma.open(autosave_file + ".tmp", "r") as f:
+        faultguard_data = FaultguardDict(pickle.load(f))
+        
+    rescue(faultguard_data)
+    return 1
+
+def start(launch, rescue, args=None, autosave_interval=None, autosave_file=None):
     """
     Start application through faultguard.
     
     Launch and rescue have access to the same dictionary. Each entry in this dictionary is stored as serialized data using the python internal 'pickle' method. The "launch" method runs in a seperate process so a fault in that process should not affect the data stored in the dictionary.
 
+    If the autosave parameters are set, the dictionary is compressed and saved in the specified time interval to the specified path. Throws an error if the autosave file already exists. After successful exit of the monitored application, the autosave file is deleted.
+
     :param launch: The applications main method. Accepts faultguard data dictionary as first and args (if not None) as second parameter.
     :param rescue: The method to call on a fault. Accepts faultguard data dictionary as first and args (if not None) as second parameter.
     :param args: Data passed to launch and rescue methods.
+    :param autosave_interval: Time in seconds between each autosave of the `faultguard` dictionary.
+    :param autosave_file: Path to file to use for autosaves.
     :returns: The applications exit code.
     """
 
+    # Ensure valid parameters
+    if autosave_interval is not None or autosave_file is not None:
+        if autosave_interval is None or autosave_file is None:
+            raise TypeError("Only one of the arguments 'autosave_interval' and 'autosave_file' is defined")
+        
+        import os
+        
+        if os.path.isfile(autosave_file):
+            raise RuntimeError("The given autosave file already exists")
+        
+        with open(autosave_file, "w") as f:
+            if not f.writable():
+                raise RuntimeError("The given autosave file is not writable")
+        
+        if os.path.isfile(autosave_file + ".tmp"):
+            os.remove(autosave_file + ".tmp")
+    
     # Detach signal handlers from faultguard process
     # Ensures faultguard does not interfere with signals like SIGINT
     orig_handlers = {}
@@ -71,17 +127,46 @@ def start(launch, rescue, args = None):
 
     # Run process
     p.start()
-    p.join()
+    
+    if autosave_interval is None:
+        p.join()
+    else:
+        # Compression library
+        import lzma
+        
+        while p.is_alive():
+            # Wait for next autosave
+            p.join(autosave_interval)
+            
+            # Autosave
+            if os.path.isfile(autosave_file + ".tmp"):
+                os.remove(autosave_file + ".tmp")
+            os.rename(autosave_file, autosave_file + ".tmp")
+            
+            with lzma.open(autosave_file, "w") as f:
+                pickle.dump(dict(managed_dict), f)
+    
+    # Close Manager process
+    # If this is not done and the faultguard process is terminated, the Manager process
+    # would keep running.
+    if p.exitcode != 0:
+        faultguard_data = FaultguardDict(dict(managed_dict))
+    manager.shutdown()
 
     # Re-attach signal handlers
     for sig in orig_handlers:
         signal.signal(sig, orig_handlers[sig])
 
     if p.exitcode != 0:
-        faultguard_data = FaultguardDict(managed_dict)
         if args is None:
             rescue(faultguard_data, p.exitcode)
         else:
             rescue(faultguard_data, p.exitcode, args)
-        
+    
+    if autosave_interval is not None and os.path.isfile(autosave_file):
+        # Remove autosave file
+        os.remove(autosave_file)
+        if os.path.isfile(autosave_file + ".tmp"):
+            os.remove(autosave_file + ".tmp")
+
     return p.exitcode

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name='faultguard',
-    version='1.0.3.post1',
+    version='1.1.0',
     py_modules=['faultguard'],
     url='https://github.com/2xB/faultguard',
     license='BSD 3-Clause License',

test_autosave.py

@@ -0,0 +1,71 @@
+from multiprocessing import Process
+import multiprocessing
+import faultguard
+import numpy as np
+import time
+import sys
+import os
+
+def launch(faultguard_data, args):
+    """
+    Demo software main method
+    
+    :param faultguard_data: Faultguard data dictionary
+    :param args: Data passed from faultguard.start.
+    """
+    print("Launching demo")
+    print(multiprocessing.current_process())
+    
+    # Some important data
+    important_data_1 = np.array([1,2,3])
+    important_data_2 = args[0] + " " + args[1]
+    
+    # Some dummy important data manipulation
+    for i in range(1):
+        important_data_1[i%3] = i
+        important_data_2 += str(i)
+        print("important_data_1:", important_data_1)
+        print("important_data_2:", important_data_2)
+        
+        # Sending important data to faultguard process
+        faultguard_data["important_data_1"] = important_data_1
+        faultguard_data["important_data_2"] = important_data_2
+        time.sleep(4)
+
+def rescue(faultguard_data, exit_code, args):
+    raise RuntimeError("Rescue handler was triggered unexpectedly...")
+
+def run_test():
+    # Run test with long autosave interval to be independent on startup times
+    faultguard.start(launch, rescue, args=("Hello", "World"), autosave_interval=1, autosave_file="test.tmp.xz")
+
+def recover(faultguard_data):
+    important_data_1 = faultguard_data["important_data_1"]
+    important_data_2 = faultguard_data["important_data_2"]
+    
+    # You might need to assign the class here by important_data_1.__class__ = ...
+    print("important_data_1:", important_data_1)
+    print("important_data_2:", important_data_2)
+    assert np.all(important_data_1 == [0, 2, 3])
+    assert important_data_2 == "Hello World0"
+
+def test_main():
+    # Prepare test environment
+    if os.path.isfile("test.tmp.xz"):
+        os.remove("test.tmp.xz")
+    
+    p = Process(target=run_test)
+    
+    # Run process
+    p.start()
+    p.join(2)
+    os.rename("test.tmp.xz", "test.tmp.xz.backup")
+    p.join()
+    
+    os.rename("test.tmp.xz.backup", "test.tmp.xz")
+    assert faultguard.recover(recover, "test.tmp.xz") == 0
+
+    os.rename("test.tmp.xz", "test.tmp.xz.tmp")
+    with open("test.tmp.xz", "w") as f:
+        f.write("Test")
+    assert faultguard.recover(recover, "test.tmp.xz") == 1

test_example.py

@@ -1,5 +1,6 @@
 import faultguard
 import numpy as np
+import os
 
 def launch(faultguard_data, args):
     """
@@ -42,7 +43,7 @@ def rescue(faultguard_data, exit_code, args):
 
     # Check if fault occurs before data was initialized
     if "important_data_1" not in faultguard_data or "important_data_2" not in faultguard_data:
-        return
+        raise RuntimeError("Faultguard dict content missing")
 
     # Restore data
     important_data_1 = faultguard_data["important_data_1"]
@@ -54,5 +55,14 @@ def rescue(faultguard_data, exit_code, args):
     assert np.all(important_data_1 == [6, 7, 5])
     assert important_data_2 == "Hello World01234567"
 
-def test_main():
+def test_main_autosave():
+    if os.path.isfile("test2.tmp.xz"):
+        os.remove("test2.tmp.xz")
+    
+    faultguard.start(launch, rescue, args=("Hello", "World"), autosave_interval=3, autosave_file="test2.tmp.xz")
+    
+    assert not os.path.isfile("test2.tmp.xz")
+    assert not os.path.isfile("test2.tmp.xz.tmp")
+    
+def test_main_no_autosave():
     faultguard.start(launch, rescue, args=("Hello", "World"))