Merge pull request #53 from aluamorim/minor_fixes

Minor fixes

Author: navjotk

examples/use_modernised.py

@@ -1,6 +1,6 @@
 import pyrevolve as pr
 import numpy as np
-import collections
+from collections.abc import Mapping
 
 
 class Symbol(object):
@@ -30,7 +30,7 @@ def __init__(self, u, m):
         self.m = m
 
     def apply(self, t_start, t_end):
-        print((">"*(t_end-t_start)).rjust(t_end))
+        # print((">"*(t_end-t_start)).rjust(t_end))
         for i in range(t_start, t_end):
             u.data = u.data + m.data
 
@@ -42,7 +42,7 @@ def __init__(self, u, m, v):
         self.m = m
 
     def apply(self, t_start, t_end):
-        print(("<"*(t_end-t_start)).rjust(t_end))
+        # print(("<"*(t_end-t_start)).rjust(t_end))
         for i in range(t_end, t_start, -1):
             v.data = v.data + m.data
 
@@ -55,7 +55,7 @@ def __init__(self, symbols):
         stores only a reference to the symbols that are passed into it.
         The symbols must be passed as a mapping symbolname->symbolobject."""
 
-        if(isinstance(symbols, collections.Mapping)):
+        if(isinstance(symbols, Mapping)):
             self.symbols = symbols
         else:
             raise Exception("Symbols must be a Mapping, for example a \

pyrevolve/profiling.py

@@ -52,7 +52,7 @@ def get_dict(self):
         results = {}
         for s_n, s_dict in self.timings.items():
             for a_n, a_time in s_dict.items():
-                results['%s_%s_timing' % (s_n, a_n)] = a_time
+                results['%s_%s_timing' % (s_n, a_n)] = "{:.2f}".format(a_time)
 
         for s_n, s_dict in self.counts.items():
             for a_n, a_time in s_dict.items():

pyrevolve/pyrevolve.py

@@ -171,6 +171,10 @@ def addByteStorage(self, compression_params):
     def makespan(self):
         return 0
 
+    @property
+    def ratio(self):
+        return 0
+
     def apply_forward(self):
         """Executes only the forward computation while storing checkpoints,
         then returns."""
@@ -268,6 +272,11 @@ def load_checkpoint(self, st_idx=0):
     def remove_checkpoint(self, st_idx=0):
         return NotImplemented
 
+    def storage_ckps(self, k=0):
+        """Returns a list of all checkpoint keys stored at the k-th
+        storage level"""
+        return NotImplemented
+
 
 class SingleLevelRevolver(BaseRevolver):
     """
@@ -345,6 +354,16 @@ def __init__(
             self.compression_params = compression_params
             self.addNumpyStorage(compression_params)
 
+    @property
+    def ratio(self):
+        return self.scheduler.ratio
+
+    def storage_ckps(self, k=0):
+        """Returns a list of all checkpoint keys stored at the k-th
+        storage level"""
+        # single level always uses first storage object on storage_list
+        return self.scheduler.storage(0)
+
 
 class MultiLevelRevolver(BaseRevolver):
     """
@@ -389,7 +408,6 @@ def __init__(
             checkpoint:         checkpoint object
             fwd_operator:       forward operator
             rev_operator:       backward operator
-            n_checkpoints:      number of checkpoints
             n_timesteps:        number of timesteps
             timings:            timings
             profiler:           profiler
@@ -443,6 +461,15 @@ def reload_scheduler(self, uf=1, ub=1, up=1):
     def makespan(self):
         return self.scheduler.makespan
 
+    @property
+    def ratio(self):
+        return self.scheduler.ratio
+
+    def storage_ckps(self, k=0):
+        """Returns a list of all checkpoint keys stored at the k-th
+        storage level"""
+        return self.scheduler.storage(k)
+
     def save_checkpoint(self, st_idx=0):
         data_pointers = self.checkpoint.get_data(self.scheduler.capo)
         self.storage_list[st_idx].push(data_pointers)

pyrevolve/schedulers/base.py

@@ -102,8 +102,8 @@ def __repr__(self):
             dict(
                 {
                     "type": self.type_names[self.type],
-                    "capo": self.capo,
-                    "old_capo": self.old_capo,
+                    "from": self.old_capo,
+                    "to": self.capo,
                     "ckp": self.ckp,
                 }
             )
@@ -143,3 +143,7 @@ def old_capo(self):
     @property
     def cp_pointer(self):
         return 0
+
+    @property
+    def oplist(self):
+        return None

pyrevolve/schedulers/crevolve.py

@@ -35,8 +35,30 @@ class CRevolve(Scheduler):
 
     def __init__(self, n_checkpoints, n_timesteps):
         super().__init__(n_checkpoints, n_timesteps)
-        self.revolve = cr.CRevolve(n_checkpoints, n_timesteps, None)
         self.__revstart_action = None
+        self.revolve = cr.CRevolve(n_checkpoints, n_timesteps, None)
+        self.__oplist = []
+        self.__stored_ckps = []
+        self.__ratio = self.__calc_ratio()
+        self.revolve = cr.CRevolve(n_checkpoints, n_timesteps, None)
+
+    def __calc_ratio(self):
+        fcomp = 0
+        ca = self.next()
+        self.__oplist.append(ca)
+        while ca.type != Action.TERMINATE:
+            if (ca.type == Action.ADVANCE) or (ca.type == Action.LASTFW):
+                st = ca.old_capo
+                end = ca.capo
+                fcomp += (end-st)
+            ca = self.next()
+            self.__oplist.append(ca)
+
+        return (fcomp/self.n_timesteps)
+
+    @property
+    def oplist(self):
+        return self.__oplist
 
     def next(self):
         if self.__revstart_action is None:
@@ -53,6 +75,8 @@ def next(self):
                     old_capo=self.old_capo,
                     ckp=self.cp_pointer,
                 )
+            if ca.type is Action.TAKESHOT:
+                self.__stored_ckps.append(ca.capo)
         else:
             ca = self.__revstart_action
             self.__revstart_action = None
@@ -69,3 +93,12 @@ def old_capo(self):
     @property
     def cp_pointer(self):
         return self.revolve.check
+
+    @property
+    def ratio(self):
+        return self.__ratio
+
+    def storage(self, k):
+        """Returns a list of all checkpoint keys stored at the k-th
+        storage level. For CRevolve, k is always 0 """
+        return self.__stored_ckps

pyrevolve/schedulers/hrevolve.py

@@ -662,6 +662,10 @@ def __init__(self, n_checkpoints, n_timesteps, architecture=None, uf=1, ub=1, up
     def resetSequence(self):
         self.__copindex = 0
 
+    @property
+    def oplist(self):
+        return self.__oplist
+
     def next(self):
         if self.__copindex >= self.n_ops:
             ha = HAction(action_type=Action.TERMINATE)
@@ -729,6 +733,14 @@ def __check_for_cpdel_condition(self):
 
         return ret
 
+    def storage(self, k):
+        """Returns a list of all checkpoint keys stored at the k-th
+        storage level"""
+        if k < self.architecture.nblevels:
+            return self.__sequence.storage[k]
+        else:
+            return None
+
     @property
     def capo(self):
         return self.__capo
@@ -745,6 +757,20 @@ def cp_pointer(self):
     def makespan(self):
         return self.__sequence.makespan
 
+    @property
+    def ratio(self):
+        # compute recomputation ratio:
+        fcomp = 0
+        for op in self.__oplist:
+            if op.type == "Forwards":
+                st = op.index[0]
+                end = op.index[1]
+                fcomp += (end-st)
+            elif op.type == "Forward":
+                fcomp += 1
+
+        return (fcomp/self.n_timesteps)
+
     def hrevolve_aux(self, l, K, cmem, hoptp=None, hopt=None):
         """
             This function is a copy of the orginal HRevolve_Aux

pyrevolve/storage.py

@@ -54,6 +54,7 @@ def __init__(self, size_ckp, n_ckp, dtype, profiler, wd=0, rd=0, name="storage")
         self.name = name
         self.__maxsize = self.size_ckp * n_ckp
         self.__current_size = 0
+        self.__itemsize = np.dtype(self.dtype).itemsize
 
         # stack interface controls
         self.__stack_ptr = -1
@@ -97,12 +98,17 @@ def isEmpty(self):
 
     @property
     def maxsize(self):
-        """Returns the maximum storage size in bytes"""
+        """Returns the maximum storage size in words of 'dtype'"""
         return self.__maxsize
 
+    @property
+    def maxsize_in_bytes(self):
+        """Returns the maximum storage size in bytes"""
+        return self.__maxsize*self.__itemsize
+
     @property
     def size(self):
-        """Returns the current storage size in bytes"""
+        """Returns the current storage size in words of 'dtype'"""
         return self.__current_size
 
     @property
@@ -216,12 +222,13 @@ def save(self, key, data_pointers):
             else:
                 ckpfile = self.datFileName + (".k%d" % (key))
                 self.checkFilesDir()
-                slot = open(ckpfile, "bw+")
+                slot = open(ckpfile, "bw")
 
             for ptr in data_pointers:
                 assert ptr.strides[-1] == ptr.itemsize
                 with self.profiler.get_timer("storage", "flatten"):
                     data = ptr.ravel()
+                    data = data.astype(self.dtype)
                 data.tofile(slot)
                 slot.flush()
                 self.__current_size += self.size_ckp
@@ -238,7 +245,7 @@ def load(self, key, locations):
             else:
                 ckpfile = self.datFileName + (".k%d" % (key))
                 self.checkFilesDir()
-                slot = open(ckpfile, "br+")
+                slot = open(ckpfile, "br")
 
             offset = 0
             for shape, ptr in zip(self.shapes[key], locations):
@@ -283,6 +290,7 @@ def save(self, key, data_pointers):
                 data = ptr.ravel()
             with self.profiler.get_timer("storage", "copy_save"):
                 np.copyto(slot[offset:(len(data) + offset)], data)
+
             offset += len(data)
             shapes.append(ptr.shape)
         self.shapes[key] = shapes

pyrevolve/tools.py

@@ -48,6 +48,7 @@ def stop(self):
         """
         Stop capturing the stream data and save the text in `capturedtext`.
         """
+
         # Print the escape character to make the readOutput method stop:
         self.origstream.write(self.escape_char)
         # Flush the stream to make sure all our data goes in before