added save to dropbox

FedeClaudi · FedeClaudi · commit cd20eac1131b · 2020-09-23T21:55:53.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
+proj/secrets.py
+
 .vscode/*
 .vscode/
 .vscode
diff --git a/proj/environment/manager.py b/proj/environment/manager.py
@@ -9,10 +9,11 @@
 from fcutils.file_io.io import save_yaml
 
 
-from proj.utils import timestamp
+from proj.utils.misc import timestamp
 from proj import paths
 from proj.animation.animate import animate_from_images
 from proj.plotting.results import plot_results
+from proj.utils.dropbox import DropBoxUtils, upload_folder
 
 FORMAT = "%(message)s"
 logging.basicConfig(
@@ -136,6 +137,13 @@ def _save_video(self):
             except (FileNotFoundError, PermissionError):
                 print("could not remove frames folder")
 
+    def _upload_to_dropbox(self):
+        dbx = DropBoxUtils()
+        dpx_path = self.datafolder.name
+        logging.info(f"Uploading data to dropbox at: {dpx_path}")
+
+        upload_folder(dbx, self.datafolder, dpx_path)
+
     def conclude(self):
         self._save_results()
         self._save_video()
@@ -146,3 +154,7 @@ def conclude(self):
             plot_every=self.plot_every,
             save_path=self.datafolder / "outcome",
         )
+
+        # Upload results to dropbox
+        if self.winstor:
+            self._upload_to_dropbox()
diff --git a/proj/environment/plotter.py b/proj/environment/plotter.py
@@ -6,7 +6,6 @@
 from fcutils.plotting.colors import desaturate_color
 from fcutils.plotting.plot_elements import plot_line_outlined
 
-from proj.utils import salmon
 from proj.animation import variables_colors as colors
 
 
@@ -54,7 +53,7 @@ def _plot_xy(self, ax, curr_goals):
                 curr_goals[:, 0],
                 curr_goals[:, 1],
                 lw=10,
-                color=salmon,
+                color="r",
                 zorder=-1,
                 solid_capstyle="round",
             )
diff --git a/proj/environment/trajectories.py b/proj/environment/trajectories.py
@@ -14,7 +14,7 @@
 )
 from fcutils.maths.filtering import line_smoother
 
-from proj.utils import interpolate_nans
+from proj.utils.misc import interpolate_nans
 
 
 def complete_given_xy(x, y, params, planning_params):
@@ -36,7 +36,9 @@ def complete_given_xy(x, y, params, planning_params):
     )
 
 
-def compute_trajectory_stats(trajectory, duration, planning_params):
+def compute_trajectory_stats(
+    trajectory, duration, planning_params, min_dist_travelled=150
+):
     # Compute stats
     n_points = len(trajectory)
     distance_travelled = np.sum(
@@ -96,15 +98,21 @@ def compute_trajectory_stats(trajectory, duration, planning_params):
     print(table)
 
     if waypoint_density < 2 or waypoint_density > 3:
-        print(
-            f"[bold red]Waypoint density of {round(waypoint_density, 3)} out of range, it should be 2 < wp < 3!"
+        log.info(
+            f"[bold red]Waypoint density of {round(waypoint_density, 3)} out of range, it should be 2 < wp < 3!",
+            extra={"markdown": True},
         )
 
     if perc_lookahead < 0.05:
-        print(
-            f"[bold red]Lookahead of {lookahead} is {perc_lookahead} of the # of waypoints, that might be too low. Values closer to 5% are advised."
+        log.info(
+            f"[bold red]Lookahead of {lookahead} is {perc_lookahead} of the # of waypoints, that might be too low. Values closer to 5% are advised.",
+            extra={"markdown": True},
         )
 
+    if distance_travelled < min_dist_travelled:
+        log.warning("Distance travelled below minimal requirement, erroring")
+        raise ValueError("Distance travelled below minimal requirement")
+
     return trajectory, duration
 
 
diff --git a/proj/environment/world.py b/proj/environment/world.py
@@ -4,7 +4,7 @@
 from fcutils.maths.geometry import calc_distance_between_points_2d
 
 from proj.environment.plotter import Plotter
-from proj.utils import polar_to_cartesian
+from proj.utils.misc import polar_to_cartesian
 
 _xy = namedtuple("xy", "x, y")
 _xyt = namedtuple("xyt", "x, y, t")
diff --git a/proj/model/model.py b/proj/model/model.py
@@ -14,7 +14,7 @@
 from fcutils.maths.geometry import calc_distance_between_points_2d
 
 from proj.model.config import Config
-from proj.utils import merge
+from proj.utils.misc import merge
 from proj.control.utils import fit_angle_in_range
 from proj.model.fast import (
     fast_dqdt,
diff --git a/proj/paths.py b/proj/paths.py
@@ -10,6 +10,8 @@
     frames_cache = Path(
         "/Users/federicoclaudi/Dropbox (UCL - SWC)/Rotation_vte/Locomotion/control/frames_cache"
     )
+
+    db_app = "/Users/federicoclaudi/Dropbox (UCL - SWC)/Apps/loco_upload"
 else:
     trials_cache = "D:\\Dropbox (UCL - SWC)\\Rotation_vte\\Locomotion\\control\\behav_data\\m46_cache.h5"
 
@@ -20,9 +22,12 @@
         "D:\\Dropbox (UCL - SWC)\\Rotation_vte\\Locomotion\\control\\frames_cache"
     )
 
+    raise NotImplementedError("db app missing")
+
 
 # winstor paths
 winstor_main = Path("/nfs/winstor/branco/Federico/Locomotion/control")
 winstor_trial_cache = (
     "/nfs/winstor/branco/Federico/Locomotion/control/m46_cache.h5"
 )
+winstor_db_app = ""
diff --git a/proj/plotting/results.py b/proj/plotting/results.py
@@ -7,7 +7,7 @@
 from fcutils.plotting.plot_elements import plot_line_outlined
 from fcutils.maths.utils import derivative
 
-from proj.utils import load_results_from_folder
+from proj.utils.misc import load_results_from_folder
 from proj.animation import variables_colors as colors
 
 
diff --git a/proj/plotting/trajectories.py b/proj/plotting/trajectories.py
@@ -1,7 +1,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from proj.utils import polar_to_cartesian
+from proj.utils.misc import polar_to_cartesian
 
 
 def plot_trajectory_polar(traj):
diff --git a/proj/utils/__init__.py b/proj/utils/__init__.py
diff --git a/proj/utils/dropbox.py b/proj/utils/dropbox.py
@@ -0,0 +1,92 @@
+import dropbox
+from dropbox.files import WriteMode
+from dropbox.exceptions import ApiError, AuthError
+
+from pathlib import Path
+
+from proj.secrets import DB_TOKEN
+from proj.paths import db_app
+
+
+def upload_folder(dbx, fld, base):
+    fld = Path(fld)
+    base = Path(base)
+
+    # loop subfolders
+    for subf in fld.glob("*"):
+        if not subf.is_dir():
+            continue
+        upload_folder(dbx, subf, base / subf.name)
+
+    for f in fld.glob("*.*"):
+        if not f.is_file():
+            continue
+
+        dest = base / f.name
+        dbx.upload_file(f, dest)
+
+
+class DropBoxUtils:
+    def __init__(self):
+        # log in
+        try:
+            self.db = dropbox.Dropbox(DB_TOKEN)
+        except AuthError:
+            raise ValueError("Failed to access dropbox app")
+
+        # keep ref to path where db app is stored locally
+        self.local = Path(db_app)
+        if not self.local.exists():
+            raise FileNotFoundError("Local path not found")
+
+    def _fix_path(self, path):
+        return str(self.local / path)
+
+    def _check_path(self, path):
+        path = str(path)
+        if path[0] != "/":
+            return "/" + path
+        else:
+            return path
+
+    def download_file(self, filepath, dest_path=None):
+        """
+            Downloads a file from self.local/filepath.
+            if dest_path is not None the file is saved there
+        """
+        if dest_path is None:
+            try:
+                self.db.files_download(self._fix_path(filepath))
+            except ApiError as e:
+                raise ValueError(f"Failed to download {filepath}: {e}")
+        else:
+            raise NotImplementedError
+
+    def upload_file(self, source, dest):
+        try:
+            with open(self._fix_path(source), "rb") as f:
+                self.db.files_upload(
+                    f.read(),
+                    self._check_path(dest),
+                    mode=WriteMode("overwrite"),
+                    autorename=True,
+                )
+        except ApiError as e:
+            raise ValueError(f"Failed to upload file {source}\n\n {e}")
+
+    # def upload_folder(self, source, dest):
+    #     source = Path(source)
+    #     dest = Path(dest)
+
+    #     # loop subfolders
+    #     for subf in source.glob('*'):
+    #         if not subf.is_dir(): continue
+    #         print(subf)
+    #         self.upload_folder(subf, dest / subf.name)
+
+    # # loop files
+    # for f in source.glob('*.*'):
+    #     if not f.is_file(): continue
+
+    #     dest = dest / f.name
+    #     self.upload_file(f, dest)
diff --git a/proj/utils/misc.py b/proj/utils/misc.py
@@ -116,8 +116,3 @@ def interpolate_nans(arr):
 
 def timestamp():
     return time.strftime("%y%m%d_%H%M%S")
-
-
-# ---------------------------------- COLORS ---------------------------------- #
-salmon = "#FA8072"
-seagreen = "#2E8B57"
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -1,6 +1,6 @@
 import numpy as np
 
-from proj.utils import cartesian_to_polar, polar_to_cartesian
+from proj.utils.misc import cartesian_to_polar, polar_to_cartesian
 
 
 def test_coord_transf():
diff --git a/workspace.py b/workspace.py
@@ -1,13 +1,37 @@
 # %%
-from rich.syntax import Syntax
-from rich import print
+from proj.utils.dropbox import DropBoxUtils
 
-with open("proj/model/config.py") as f:
-    content = f.read()
+db = DropBoxUtils()
 
-content
 
 # %%
-print(Syntax(content, "python", theme="monokai", line_numbers=True))
+from pathlib import Path
+
+fld = Path("/Users/federicoclaudi/Documents/Github/pysical_locomotion/proj")
+
+
+# %%
+def upload_folder(dbx, fld, base):
+    fld = Path(fld)
+    base = Path(base)
+
+    # loop subfolders
+    for subf in fld.glob("*"):
+        if not subf.is_dir():
+            continue
+        upload_folder(dbx, subf, base / subf.name)
+
+    for f in fld.glob("*.*"):
+        if not f.is_file():
+            continue
+
+        dest = base / f.name
+        dbx.upload_file(f, dest)
+
+
+base = ""
+upload_folder(db, fld, "test")
+# %%
+db.upload_folder(fld, "test2")
 
 # %%

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+proj/secrets.py`
	`2`	`+`
`1`	`3`	`.vscode/*`
`2`	`4`	`.vscode/`
`3`	`5`	`.vscode`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,8 @@`
`10`	`10`	`frames_cache = Path(`
`11`	`11`	`"/Users/federicoclaudi/Dropbox (UCL - SWC)/Rotation_vte/Locomotion/control/frames_cache"`
`12`	`12`	`)`
	`13`	`+`
	`14`	`+ db_app = "/Users/federicoclaudi/Dropbox (UCL - SWC)/Apps/loco_upload"`
`13`	`15`	`else:`
`14`	`16`	`trials_cache = "D:\\Dropbox (UCL - SWC)\\Rotation_vte\\Locomotion\\control\\behav_data\\m46_cache.h5"`
`15`	`17`
`@@ -20,9 +22,12 @@`
`20`	`22`	`"D:\\Dropbox (UCL - SWC)\\Rotation_vte\\Locomotion\\control\\frames_cache"`
`21`	`23`	`)`
`22`	`24`
	`25`	`+ raise NotImplementedError("db app missing")`
	`26`	`+`
`23`	`27`
`24`	`28`	`# winstor paths`
`25`	`29`	`winstor_main = Path("/nfs/winstor/branco/Federico/Locomotion/control")`
`26`	`30`	`winstor_trial_cache = (`
`27`	`31`	`"/nfs/winstor/branco/Federico/Locomotion/control/m46_cache.h5"`
`28`	`32`	`)`
	`33`	`+winstor_db_app = ""`