better logging

Author: FedeClaudi

proj/__init__.py

@@ -1,9 +1,5 @@
 # Set up logging
-import logging
-
-for module in ["matplotlib", "pandas", "numpy"]:
-    requests_logger = logging.getLogger("matplotlib")
-    requests_logger.setLevel(logging.ERROR)
+from proj.utils.logging import log
 
 # import stuff
 from proj.model.config import Config

proj/environment/manager.py

@@ -1,26 +1,21 @@
 import shutil
 import numpy as np
 import pandas as pd
-from fancylog import fancylog
 import logging
+import json
 
 from rich.logging import RichHandler
 
 from fcutils.file_io.io import save_yaml
 
 
 from proj.utils.misc import timestamp
-from proj import paths
+from proj import paths, log
 from proj.animation.animate import animate_from_images
 from proj.plotting.results import plot_results
 from proj.utils.dropbox import DropBoxUtils, upload_folder
 from proj.utils.slack import send_slack_message
 
-FORMAT = "%(message)s"
-logging.basicConfig(
-    level="NOTSET", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()]
-)
-
 
 class Manager:
     def __init__(self, model, winstor=False):
@@ -62,20 +57,10 @@ def __init__(self, model, winstor=False):
         self._start_logging()
 
     def _start_logging(self):
-        # Start logging
-        fancylog.start_logging(
-            output_dir=str(self.datafolder),
-            filename=self.exp_name + ".log",
-            multiprocessing_aware=False,
-            write_git=False,
-            verbose=False,
-            write_cli_args=False,
-            file_log_level="INFO",
-        )
-
-        # log main folder
-        log = logging.getLogger("rich")
-        log.setLevel(logging.INFO)
+        filename = str(self.datafolder / f"{self.exp_name}.log")
+        fh = logging.FileHandler(filename)
+        fh.setFormatter(RichHandler())
+        log.addHandler(fh)
 
         log.info(
             f"[bold green] Saving data at: {self.datafolder}",
@@ -84,12 +69,8 @@ def _start_logging(self):
 
     def _log_conf(self):
         # log config.py
-        try:
-            with open("proj/model/config.py") as f:
-                conf = "\n" + f.read()
-        except FileNotFoundError:
-            conf = self.model.config_dict()
-        logging.info(conf)
+        conf = json.dumps(self.model.config_dict(), sort_keys=True, indent=4)
+        log.info("Config parameters:\n" + conf, extra={"markup": True})
 
     def _save_results(self):
         # save config
@@ -145,7 +126,9 @@ def _save_video(self):
     def _upload_to_dropbox(self):
         dbx = DropBoxUtils()
         dpx_path = self.datafolder.name
-        logging.info(f"Uploading data to dropbox at: {dpx_path}")
+        log.info(
+            f"Uploading data to dropbox at: {dpx_path}", extra={"markup": True}
+        )
 
         upload_folder(dbx, self.datafolder, dpx_path)
 
@@ -170,13 +153,13 @@ def conclude(self):
 
         # Upload results to dropbox
         if self.winstor:
-            logging.info("Uploading to dropbox")
+            log.info("Uploading to dropbox", extra={"markup": True})
             try:
                 self._upload_to_dropbox()
             except Exception as e:
                 logging.error(f"Failed to upload to dropbox: {e}")
 
-            logging.info("Sending slack message")
+            log.info("Sending slack message", extra={"markup": True})
             send_slack_message(
                 f"""
                 \n
@@ -187,10 +170,10 @@ def conclude(self):
                 """
             )
         else:
-            logging.info("Did not upload to dropbox")
+            log.info("Did not upload to dropbox", extra={"markup": True})
 
     def failed(self):
-        logging.info("Sending slack FAILED message")
+        log.info("Sending slack FAILED message", extra={"markup": True})
         if self.winstor:
             send_slack_message(
                 f"""

proj/model/config.py

@@ -68,7 +68,7 @@ class Config:
         distance=150,
         max_speed=100,
         min_speed=80,
-        min_dist=5,  # if agent is within this distance from trajectory end the goal is considered achieved
+        min_dist=0,  # if agent is within this distance from trajectory end the goal is considered achieved
         # ? for trajectories from data
         px_to_cm=1 / 30.8,  # convert px values to cm
         # dist_th=60,  # keep frames only after moved away from start location

proj/run/runner.py

@@ -8,6 +8,7 @@
 from rich import print
 from rich.text import Text
 import logging
+import traceback
 
 
 class SpeedColumn(TextColumn):
@@ -112,11 +113,14 @@ def run_experiment(
                 if environment.stop:
                     log.info("environment says STOP")
                     break
+
             except Exception as e:
                 logging.error(
-                    f"Failed to take next step in simulation.\n error {e}"
+                    f"Failed to take next step in simulation.\nError: {e}\n\n"
                 )
-                break
+                logging.error(traceback.print_exc())
+                environment.failed()
+                return
 
     log.info(f"Terminated after {itern} iterations.")
 

proj/utils/logging.py

@@ -0,0 +1,31 @@
+import logging
+from rich.logging import RichHandler
+
+# supress warnings
+import pandas as pd
+import warnings
+
+warnings.simplefilter(action="ignore", category=pd.errors.PerformanceWarning)
+
+# Set up RICH logger
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s",
+    datefmt="[%X]",
+    handlers=[RichHandler(rich_tracebacks=True)],
+)
+
+log = logging.getLogger("rich")
+
+
+# Disable logging some packages
+for module in [
+    "matplotlib",
+    "pandas",
+    "numpy",
+    "tensorflow",
+    "pandas",
+    "tables",
+]:
+    logger = logging.getLogger(module)
+    logger.setLevel(logging.ERROR)

setup.py

@@ -18,6 +18,7 @@
     "fancylog",
     "slackclient",
     "dropbox",
+    "json",
 ]
 
 setup(

workspace.py

@@ -1,83 +0,0 @@
-# %%
-
-
-from behaviour.tracking.tracking import prepare_tracking_data
-from fcutils.maths.geometry import (
-    calc_angle_between_vectors_of_points_2d as get_bone_angle,
-)
-import numpy as np
-import pandas as pd
-
-# %%
-path = "/Users/federicoclaudi/Dropbox (UCL - SWC)/Rotation_vte/Locomotion/control/behav_data/Zane/ZM_200913_ZM006_videoDeepCut_resnet50_DLCSep6shuffle1_200000.h5"
-
-
-tracking = prepare_tracking_data(
-    path, smooth_dir_mvmt=False, likelihood_th=0.8, interpolate_nans=True
-)
-
-
-body_orientation = get_bone_angle(
-    tracking["body"].x.values,
-    tracking["body"].y.values,
-    tracking["tail_base"].x.values,
-    tracking["tail_base"].y.values,
-)
-
-
-# %%
-escapes_file = "/Users/federicoclaudi/Dropbox (UCL - SWC)/Rotation_vte/Locomotion/control/behav_data/Zane/escapes.txt"
-with open(escapes_file, "r") as f:
-    escapes = [int(e) for e in f.readlines()]
-
-# %%
-trials = {}
-for bp in tracking.keys():
-    trials[bp + "_xy"] = []
-    trials[bp + "_speed"] = []
-
-    if bp == "body":
-        trials[bp + "_orientation"] = []
-
-
-for escape in escapes:
-    for bp in tracking.keys():
-        start = escape
-        end = start + 1000
-        x = tracking[bp].x[start:end]
-        y = tracking[bp].y[start:end]
-        s = tracking[bp].speed[start:end]
-
-        trials[bp + "_xy"].append(np.vstack([x, y]).T)
-        trials[bp + "_speed"].append(s)
-
-        if bp == "body":
-            trials["body_orientation"].append(body_orientation[start:end])
-
-
-trials = pd.DataFrame(trials)
-
-# %%
-import matplotlib.pyplot as plt
-
-f, ax = plt.subplots()
-
-for i, trial in trials.iterrows():
-    angle = np.unwrap(np.radians(90 - trial.body_orientation))
-    ax.scatter(trial.body_xy[:, 0], trial.body_xy[:, 1], c=angle)
-
-ax.axis("equal")
-
-# %%
-f, ax = plt.subplots()
-for bp in tracking.keys():
-    ax.plot(tracking[bp].x[5000:5500], tracking[bp].y[5000:5500])
-
-# %%
-trials.to_hdf(
-    "/Users/federicoclaudi/Dropbox (UCL - SWC)/Rotation_vte/Locomotion/control/behav_data/zanes.h5",
-    key="hdf",
-)
-
-
-# %%