Less memory-hungry rename

Author: JasonGross

notebooks_jason/wandb_models_rename.py

@@ -12,12 +12,12 @@
 model_base.mkdir(exist_ok=True, parents=True)
 known_models = set()
 for path in tqdm(list(model_base.glob("*.pth")), "Known models"):
-    known_models.add(path.read_bytes())
+    known_models.add(hash(path.read_bytes()))
 
 with tqdm(wandbs) as pbar:
     for path in pbar:
         pbar.set_postfix({})
-        if path.read_bytes() in known_models:
+        if hash(path.read_bytes()) in known_models:
             continue
         cache = torch.load(path, map_location="cpu")
         # if cache["run_config"]["experiment"]["p"] not in (7, 12):