From a78281097d0a56ffb05764eef53db339c3697339 Mon Sep 17 00:00:00 2001 From: ai-mg Date: Wed, 19 Feb 2025 18:29:34 +0100 Subject: [PATCH 1/3] Update conversion script to skip missing files --- tools/convert_crowdhuman_to_coco.py | 109 ++++++++++++++++++---------- 1 file changed, 72 insertions(+), 37 deletions(-) diff --git a/tools/convert_crowdhuman_to_coco.py b/tools/convert_crowdhuman_to_coco.py index 62e0b667..fc097c59 100644 --- a/tools/convert_crowdhuman_to_coco.py +++ b/tools/convert_crowdhuman_to_coco.py @@ -3,55 +3,90 @@ import json from PIL import Image -DATA_PATH = 'datasets/crowdhuman/' -OUT_PATH = DATA_PATH + 'annotations/' -SPLITS = ['val', 'train'] +DATA_PATH = "datasets/crowdhuman/" +OUT_PATH = os.path.join(DATA_PATH, "annotations") +SPLITS = ["val", "train"] DEBUG = False + def load_func(fpath): - print('fpath', fpath) - assert os.path.exists(fpath) - with open(fpath,'r') as fid: + print("Loading annotation file:", fpath) + assert os.path.exists(fpath), f"Annotation file not found: {fpath}" + with open(fpath, "r") as fid: lines = fid.readlines() - records =[json.loads(line.strip('\n')) for line in lines] + records = [json.loads(line.strip("\n")) for line in lines] return records -if __name__ == '__main__': + +if __name__ == "__main__": if not os.path.exists(OUT_PATH): os.mkdir(OUT_PATH) for split in SPLITS: - data_path = DATA_PATH + split - out_path = OUT_PATH + '{}.json'.format(split) - out = {'images': [], 'annotations': [], 'categories': [{'id': 1, 'name': 'person'}]} - ann_path = DATA_PATH + 'annotation_{}.odgt'.format(split) + data_path = os.path.join(DATA_PATH, split) + out_path = os.path.join(OUT_PATH, "{}.json".format(split)) + out = { + "images": [], + "annotations": [], + "categories": [{"id": 1, "name": "person"}], + } + ann_path = os.path.join(DATA_PATH, "annotation_{}.odgt".format(split)) anns_data = load_func(ann_path) image_cnt = 0 ann_cnt = 0 - video_cnt = 0 + for ann_data in anns_data: + file_path = os.path.join( + DATA_PATH, + "CrowdHuman_{}".format(split), + "{}.jpg".format(ann_data["ID"]), + ) + if not os.path.exists(file_path): + print("Skipping missing file:", file_path) + continue # Skip this annotation if the file is not found + try: + im = Image.open(file_path) + except Exception as e: + print("Error opening file {}: {}".format(file_path, e)) + continue + image_cnt += 1 - file_path = DATA_PATH + 'CrowdHuman_{}/'.format(split) + '{}.jpg'.format(ann_data['ID']) - im = Image.open(file_path) - image_info = {'file_name': '{}.jpg'.format(ann_data['ID']), - 'id': image_cnt, - 'height': im.size[1], - 'width': im.size[0]} - out['images'].append(image_info) - if split != 'test': - anns = ann_data['gtboxes'] - for i in range(len(anns)): + image_info = { + "file_name": "{}.jpg".format(ann_data["ID"]), + "id": image_cnt, + "height": im.size[1], + "width": im.size[0], + } + out["images"].append(image_info) + + if split != "test": + anns = ann_data.get("gtboxes", []) + for ann in anns: ann_cnt += 1 - fbox = anns[i]['fbox'] - ann = {'id': ann_cnt, - 'category_id': 1, - 'image_id': image_cnt, - 'track_id': -1, - 'bbox_vis': anns[i]['vbox'], - 'bbox': fbox, - 'area': fbox[2] * fbox[3], - 'iscrowd': 1 if 'extra' in anns[i] and \ - 'ignore' in anns[i]['extra'] and \ - anns[i]['extra']['ignore'] == 1 else 0} - out['annotations'].append(ann) - print('loaded {} for {} images and {} samples'.format(split, len(out['images']), len(out['annotations']))) - json.dump(out, open(out_path, 'w')) \ No newline at end of file + fbox = ann["fbox"] + annotation = { + "id": ann_cnt, + "category_id": 1, + "image_id": image_cnt, + "track_id": -1, + "bbox_vis": ann["vbox"], + "bbox": fbox, + "area": fbox[2] * fbox[3], + "iscrowd": ( + 1 + if ( + "extra" in ann + and "ignore" in ann["extra"] + and ann["extra"]["ignore"] == 1 + ) + else 0 + ), + } + out["annotations"].append(annotation) + + print( + "Loaded {}: {} images and {} annotations".format( + split, len(out["images"]), len(out["annotations"]) + ) + ) + with open(out_path, "w") as fout: + json.dump(out, fout) From e821fa6f0e47e3954904eb9534ce6ba1a8c960d1 Mon Sep 17 00:00:00 2001 From: ai-mg Date: Wed, 19 Feb 2025 18:59:29 +0100 Subject: [PATCH 2/3] Update conversion script to skip missing files --- tools/convert_crowdhuman_to_coco.py | 88 ++++++++++++----------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/tools/convert_crowdhuman_to_coco.py b/tools/convert_crowdhuman_to_coco.py index fc097c59..9c29f728 100644 --- a/tools/convert_crowdhuman_to_coco.py +++ b/tools/convert_crowdhuman_to_coco.py @@ -3,45 +3,39 @@ import json from PIL import Image -DATA_PATH = "datasets/crowdhuman/" -OUT_PATH = os.path.join(DATA_PATH, "annotations") -SPLITS = ["val", "train"] +DATA_PATH = 'datasets/crowdhuman/' +OUT_PATH = os.path.join(DATA_PATH, 'annotations') +SPLITS = ['val', 'train'] DEBUG = False - def load_func(fpath): - print("Loading annotation file:", fpath) + print('Loading annotation file:', fpath) assert os.path.exists(fpath), f"Annotation file not found: {fpath}" - with open(fpath, "r") as fid: + with open(fpath, 'r') as fid: lines = fid.readlines() - records = [json.loads(line.strip("\n")) for line in lines] + records = [json.loads(line.strip('\n')) for line in lines] return records - -if __name__ == "__main__": +if __name__ == '__main__': if not os.path.exists(OUT_PATH): os.mkdir(OUT_PATH) for split in SPLITS: data_path = os.path.join(DATA_PATH, split) - out_path = os.path.join(OUT_PATH, "{}.json".format(split)) + out_path = os.path.join(OUT_PATH, '{}.json'.format(split)) out = { - "images": [], - "annotations": [], - "categories": [{"id": 1, "name": "person"}], + 'images': [], + 'annotations': [], + 'categories': [{'id': 1, 'name': 'person'}] } - ann_path = os.path.join(DATA_PATH, "annotation_{}.odgt".format(split)) + ann_path = os.path.join(DATA_PATH, 'annotation_{}.odgt'.format(split)) anns_data = load_func(ann_path) image_cnt = 0 ann_cnt = 0 for ann_data in anns_data: - file_path = os.path.join( - DATA_PATH, - "CrowdHuman_{}".format(split), - "{}.jpg".format(ann_data["ID"]), - ) + file_path = os.path.join(DATA_PATH, 'CrowdHuman_{}'.format(split), '{}.jpg'.format(ann_data['ID'])) if not os.path.exists(file_path): - print("Skipping missing file:", file_path) + print('Skipping missing file:', file_path) continue # Skip this annotation if the file is not found try: im = Image.open(file_path) @@ -51,42 +45,30 @@ def load_func(fpath): image_cnt += 1 image_info = { - "file_name": "{}.jpg".format(ann_data["ID"]), - "id": image_cnt, - "height": im.size[1], - "width": im.size[0], + 'file_name': '{}.jpg'.format(ann_data['ID']), + 'id': image_cnt, + 'height': im.size[1], + 'width': im.size[0] } - out["images"].append(image_info) - - if split != "test": - anns = ann_data.get("gtboxes", []) + out['images'].append(image_info) + + if split != 'test': + anns = ann_data.get('gtboxes', []) for ann in anns: ann_cnt += 1 - fbox = ann["fbox"] + fbox = ann['fbox'] annotation = { - "id": ann_cnt, - "category_id": 1, - "image_id": image_cnt, - "track_id": -1, - "bbox_vis": ann["vbox"], - "bbox": fbox, - "area": fbox[2] * fbox[3], - "iscrowd": ( - 1 - if ( - "extra" in ann - and "ignore" in ann["extra"] - and ann["extra"]["ignore"] == 1 - ) - else 0 - ), + 'id': ann_cnt, + 'category_id': 1, + 'image_id': image_cnt, + 'track_id': -1, + 'bbox_vis': ann['vbox'], + 'bbox': fbox, + 'area': fbox[2] * fbox[3], + 'iscrowd': 1 if ('extra' in ann and 'ignore' in ann['extra'] and ann['extra']['ignore'] == 1) else 0 } - out["annotations"].append(annotation) - - print( - "Loaded {}: {} images and {} annotations".format( - split, len(out["images"]), len(out["annotations"]) - ) - ) - with open(out_path, "w") as fout: + out['annotations'].append(annotation) + + print('Loaded {}: {} images and {} annotations'.format(split, len(out['images']), len(out['annotations']))) + with open(out_path, 'w') as fout: json.dump(out, fout) From e3fb6072e717706dcb14af57f13d75e914e9d178 Mon Sep 17 00:00:00 2001 From: ai-mg Date: Wed, 19 Feb 2025 19:19:17 +0100 Subject: [PATCH 3/3] Replace old uncompatible onxx versions --- requirements.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index aae46682..1fdb097d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,9 @@ filterpy h5py # verified versions -onnx==1.8.1 -onnxruntime==1.8.0 -onnx-simplifier==0.3.5 +# onnx==1.8.1 +# onnxruntime==1.8.0 +# onnx-simplifier==0.3.5 +onnx>=1.14.0 +onnxruntime>=1.14.0 +onnx-simplifier>=0.4.10