Auto format everything

Author: DaWelter

pyproject.toml

@@ -1,4 +1,4 @@
 [tool.black]
-line-length = 120
-skip-string-normalization = true
+line-length = 100
+skip-string-normalization = false
 target-version = ['py311']

scripts/add_pose_pseudolabels.py

@@ -29,58 +29,66 @@
 
 def setup_loader(args):
     ds = Hdf5PoseDataset(
-        args.filename, 
-        transform=Compose([
-            dtr.batch.offset_points_by_half_pixel, # For when pixels are considered grid cell centers
-        ]),
-        monochrome=True)
+        args.filename,
+        transform=Compose(
+            [
+                dtr.batch.offset_points_by_half_pixel,  # For when pixels are considered grid cell centers
+            ]
+        ),
+        monochrome=True,
+    )
     if args.dryrun:
         ds = Subset(ds, np.arange(10))
     N = len(ds)
-    loader = dtr.PostprocessingLoader(ds, args.batchsize,
+    loader = dtr.PostprocessingLoader(
+        ds,
+        args.batchsize,
         shuffle=False,
         num_workers=utils.num_workers(),
         postprocess=None,
-        collate_fn= lambda samples : samples,
+        collate_fn=lambda samples: samples,
     )
     return loader, ds
 
 
-def fit_batch(net : InferenceNetwork, batch : List[Batch]):
-    images = [ s['image'] for s in batch ]
-    rois = torch.stack([ s['roi'] for s in batch ])
-    indices = torch.stack([ s['index'] for s in batch ])
-    out = Predictor(net, focus_roi_expansion_factor=1.2).predict_batch(images, rois, )
-    out = {
-        k:out[k] for k in 'unnormalized_quat coord pt3d_68 shapeparam'.split()
-    }
-    out.update(index = indices)
+def fit_batch(net: InferenceNetwork, batch: List[Batch]):
+    images = [s["image"] for s in batch]
+    rois = torch.stack([s["roi"] for s in batch])
+    indices = torch.stack([s["index"] for s in batch])
+    out = Predictor(net, focus_roi_expansion_factor=1.2).predict_batch(
+        images,
+        rois,
+    )
+    out = {k: out[k] for k in "unnormalized_quat coord pt3d_68 shapeparam".split()}
+    out.update(index=indices)
     return out
 
 
 def test_quats_average():
     def positivereal(q):
-        s = np.sign(q[...,3])
-        return q*s[...,None]
+        s = np.sign(q[..., 3])
+        return q * s[..., None]
+
     from scipy.spatial.transform import Rotation
+
     expected_quats = Rotation.random(10).as_quat()
     quats = Rotation.from_quat(np.repeat(expected_quats, 10, axis=0))
-    offsets = Rotation.random(10*10).as_rotvec()*0.01
+    offsets = Rotation.random(10 * 10).as_rotvec() * 0.01
     quats = quats * Rotation.from_rotvec(offsets)
-    quats = quats.as_quat().reshape((10,10,4)).transpose(1,0,2)
+    quats = quats.as_quat().reshape((10, 10, 4)).transpose(1, 0, 2)
     out = quat_average(quats)
-    #print (positivereal(out) - positivereal(expected_quats))
-    assert np.allclose(positivereal(out) , positivereal(expected_quats), atol=0.02)
+    # print (positivereal(out) - positivereal(expected_quats))
+    assert np.allclose(positivereal(out), positivereal(expected_quats), atol=0.02)
 
 
 @torch.no_grad()
 def fitall(args):
     assert all(isfile(f) for f in args.checkpoints)
-    print ("Inferring from networks:", args.checkpoints)
+    print("Inferring from networks:", args.checkpoints)
 
-    with h5py.File(args.filename, 'r+') as f:
+    with h5py.File(args.filename, "r+") as f:
         g = f.require_group(args.hdfgroupname) if args.hdfgroupname else f
-        for key in 'coords quats pt3d_68 shapeparams':
+        for key in "coords quats pt3d_68 shapeparams":
             try:
                 del g[key]
             except KeyError:
@@ -91,22 +99,20 @@ def fitall(args):
 
     outputs_per_net = defaultdict(list)
     for modelfile in tqdm.tqdm(args.checkpoints, desc="Network"):
-        net = load_pose_network(modelfile, 'cuda')
-        outputs = [ fit_batch(net, batch) for batch in tqdm.tqdm(loader, "Batch") ]
+        net = load_pose_network(modelfile, "cuda")
+        outputs = [fit_batch(net, batch) for batch in tqdm.tqdm(loader, "Batch")]
         outputs = utils.list_of_dicts_to_dict_of_lists(outputs)
-        outputs = {k:np.concatenate(v,axis=0) for k,v in outputs.items() }
-        ordering = np.argsort(outputs.pop('index'))
-        outputs = { k:v[ordering] for k,v in outputs.items() }
-        for k,v in outputs.items():
+        outputs = {k: np.concatenate(v, axis=0) for k, v in outputs.items()}
+        ordering = np.argsort(outputs.pop("index"))
+        outputs = {k: v[ordering] for k, v in outputs.items()}
+        for k, v in outputs.items():
             outputs_per_net[k].append(v)
         del outputs
-    outputs_per_net = {
-        k:np.stack(v) for k,v in outputs_per_net.items()
-    }
+    outputs_per_net = {k: np.stack(v) for k, v in outputs_per_net.items()}
 
     del loader
     del ds
-    gc.collect() # Ensure the hdf5 file in the data was really closed.
+    gc.collect()  # Ensure the hdf5 file in the data was really closed.
     # There is no way to enforce it. We can only hope the garbage
     # collector will destroy the objects. If there is still a reference
     # left it will be read-only and lead to failure when trying to write
@@ -115,30 +121,56 @@ def fitall(args):
     # FIXME: final quats output is busted. Values are more or less garbage.
     #        unnormalized_quat looks fine!
 
-    quats = quat_average(outputs_per_net.pop('unnormalized_quat'))
-    coords = np.average(outputs_per_net.pop('coord'), axis=0)
-    pt3d_68 = np.average(outputs_per_net.pop('pt3d_68'), axis=0)
-    shapeparams = np.average(outputs_per_net.pop('shapeparam'), axis=0)
+    quats = quat_average(outputs_per_net.pop("unnormalized_quat"))
+    coords = np.average(outputs_per_net.pop("coord"), axis=0)
+    pt3d_68 = np.average(outputs_per_net.pop("pt3d_68"), axis=0)
+    shapeparams = np.average(outputs_per_net.pop("shapeparam"), axis=0)
 
     assert len(quats) == num_samples
 
-    with h5py.File(args.filename, 'r+') as f:
+    with h5py.File(args.filename, "r+") as f:
         g = f.require_group(args.hdfgroupname) if args.hdfgroupname else f
-        ds_quats = create_pose_dataset(g, C.quat, count=num_samples, data=quats, exists_ok=args.overwrite)
-        ds_coords = create_pose_dataset(g, C.xys, count=num_samples, data=coords, exists_ok=args.overwrite)
-        ds_pt3d_68 = create_pose_dataset(g, C.points, name='pt3d_68', count=num_samples, shape_wo_batch_dim=(68,3), data=pt3d_68, exists_ok=args.overwrite)
-        ds_shapeparams = create_pose_dataset(g,C.general, name='shapeparams', count=num_samples, shape_wo_batch_dim=(50,), data = shapeparams, exists_ok=args.overwrite)
-
-
-if __name__== '__main__':
+        ds_quats = create_pose_dataset(
+            g, C.quat, count=num_samples, data=quats, exists_ok=args.overwrite
+        )
+        ds_coords = create_pose_dataset(
+            g, C.xys, count=num_samples, data=coords, exists_ok=args.overwrite
+        )
+        ds_pt3d_68 = create_pose_dataset(
+            g,
+            C.points,
+            name="pt3d_68",
+            count=num_samples,
+            shape_wo_batch_dim=(68, 3),
+            data=pt3d_68,
+            exists_ok=args.overwrite,
+        )
+        ds_shapeparams = create_pose_dataset(
+            g,
+            C.general,
+            name="shapeparams",
+            count=num_samples,
+            shape_wo_batch_dim=(50,),
+            data=shapeparams,
+            exists_ok=args.overwrite,
+        )
+
+
+if __name__ == "__main__":
     test_quats_average()
     parser = argparse.ArgumentParser()
-    parser.add_argument('filename', type=str, help='the dataset to label')
-    parser.add_argument('-c','--checkpoints', help='model checkpoint', nargs='*', type=str)
-    parser.add_argument('-b','--batchsize', help="The batch size", type=int, default=512)
-    parser.add_argument('--hdf-group-name', help="Group to store the annotations in", type=str, default='', dest='hdfgroupname')
-    parser.add_argument('--dryrun', default=False, action='store_true')
-    parser.add_argument('--overwrite', '-f', default=False, action='store_true')
+    parser.add_argument("filename", type=str, help="the dataset to label")
+    parser.add_argument("-c", "--checkpoints", help="model checkpoint", nargs="*", type=str)
+    parser.add_argument("-b", "--batchsize", help="The batch size", type=int, default=512)
+    parser.add_argument(
+        "--hdf-group-name",
+        help="Group to store the annotations in",
+        type=str,
+        default="",
+        dest="hdfgroupname",
+    )
+    parser.add_argument("--dryrun", default=False, action="store_true")
+    parser.add_argument("--overwrite", "-f", default=False, action="store_true")
     args = parser.parse_args()
-    args.device = 'cuda'
-    fitall(args)
+    args.device = "cuda"
+    fitall(args)

scripts/create_aflw2k3d_closed_eyes.py

@@ -8,30 +8,37 @@
 from face3drotationaugmentation.generate import augment_eyes_only, make_sample_for_passthrough
 from face3drotationaugmentation.datasetwriter import dataset_writer
 
-deg2rad = np.pi/180.
+deg2rad = np.pi / 180.0
 
 
-def main(filename : str, outputfilename : str, max_num_frames : int, prob_closed_eyes : float):
+def main(filename: str, outputfilename: str, max_num_frames: int, prob_closed_eyes: float):
     rng = np.random.RandomState(seed=1234567)
 
     with closing(DatasetAFLW2k3D(filename)) as ds300wlp, dataset_writer(outputfilename) as writer:
         num_frames = min(max_num_frames, len(ds300wlp))
         for _, sample in tqdm.tqdm(zip(range(num_frames), ds300wlp), total=num_frames):
-            if sample['scale'] <= 0.: # TODO: actual decent validation?
-                print (f"Error: invalid head size = {sample['scale']}. Putting original sample!")
+            if sample["scale"] <= 0.0:  # TODO: actual decent validation?
+                print(f"Error: invalid head size = {sample['scale']}. Putting original sample!")
                 generated_sample = make_sample_for_passthrough(sample)
             else:
                 generated_sample = augment_eyes_only(prob_closed_eyes, rng, sample)
-            writer.write(sample['name'], generated_sample)
+            writer.write(sample["name"], generated_sample)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser("Only Eye Augmentation")
     parser.add_argument("aflw2k3d", type=str, help="zip file")
     parser.add_argument("outputfilename", type=str, help="hdf5 file")
-    parser.add_argument("-n", help="subset of n samples", type=int, default=1<<32)
-    parser.add_argument("--prob-closed-eyes", type=float, default=0., help="probability for closing eyes (between 0 and 1)")
+    parser.add_argument("-n", help="subset of n samples", type=int, default=1 << 32)
+    parser.add_argument(
+        "--prob-closed-eyes",
+        type=float,
+        default=0.0,
+        help="probability for closing eyes (between 0 and 1)",
+    )
     args = parser.parse_args()
-    if not (args.outputfilename.lower().endswith('.h5') or args.outputfilename.lower().endswith('.hdf5')):
-            raise ValueError("outputfilename must have hdf5 filename extension")
-    main(args.aflw2k3d, args.outputfilename, args.n, prob_closed_eyes=args.prob_closed_eyes)
+    if not (
+        args.outputfilename.lower().endswith(".h5") or args.outputfilename.lower().endswith(".hdf5")
+    ):
+        raise ValueError("outputfilename must have hdf5 filename extension")
+    main(args.aflw2k3d, args.outputfilename, args.n, prob_closed_eyes=args.prob_closed_eyes)

scripts/dsjoin.py

@@ -15,7 +15,9 @@ def _batched_copy(dst, src, dst_offset=0):
     n = min(dst.shape[0] + dst_offset, src.shape[0])
     for a in range(0, n, bs):
         b = min(n, a + bs)
-        dst[a + dst_offset : b + dst_offset, ...] = src[a:b, ...]  # Buffer in memory, then write. This works ...
+        dst[a + dst_offset : b + dst_offset, ...] = src[
+            a:b, ...
+        ]  # Buffer in memory, then write. This works ...
 
 
 def concatenating_join(name1: str, items: Sequence[h5py.Dataset], fout: h5py.File):
@@ -24,7 +26,9 @@ def concatenating_join(name1: str, items: Sequence[h5py.Dataset], fout: h5py.Fil
     N = sum(sizes)
     print(f"Copying {name1}: {sizes} items of type {first.dtype}")
 
-    dst = fout.create_dataset_like(name1, first, shape=(N, *first.shape[1:]), maxshape=(N,) + first.shape[1:])
+    dst = fout.create_dataset_like(
+        name1, first, shape=(N, *first.shape[1:]), maxshape=(N,) + first.shape[1:]
+    )
     assert all(list(first.attrs.items()) == list(ds.attrs.items()) for ds in items)
     copy_attributes(first, dst)
     try: