|
22 | 22 | except ImportError: |
23 | 23 | print("Please install pyav to use video processing functions.") |
24 | 24 |
|
| 25 | +def get_frame_ids(start_frame, end_frame, num_segments=32, jitter=True): |
| 26 | + frame_ids = np.convolve(np.linspace(start_frame, end_frame, num_segments + 1), [0.5, 0.5], mode='valid') |
| 27 | + if jitter: |
| 28 | + seg_size = float(end_frame - start_frame - 1) / num_segments |
| 29 | + shift = (np.random.rand(num_segments) - 0.5) * seg_size |
| 30 | + frame_ids += shift |
| 31 | + return frame_ids.astype(int).tolist() |
| 32 | + |
| 33 | +# def get_video_reader(videoname, num_threads, fast_rrc, rrc_params, fast_rcc, rcc_params): |
| 34 | +# video_reader = None |
| 35 | +# if fast_rrc: |
| 36 | +# video_reader = VideoReader( |
| 37 | +# videoname, |
| 38 | +# num_threads=num_threads, |
| 39 | +# width=rrc_params[0], height=rrc_params[0], |
| 40 | +# use_rrc=True, scale_min=rrc_params[1][0], scale_max=rrc_params[1][1], |
| 41 | +# ) |
| 42 | +# elif fast_rcc: |
| 43 | +# video_reader = VideoReader( |
| 44 | +# videoname, |
| 45 | +# num_threads=num_threads, |
| 46 | +# width=rcc_params[0], height=rcc_params[0], |
| 47 | +# use_rcc=True, |
| 48 | +# ) |
| 49 | +# else: |
| 50 | +# video_reader = VideoReader(videoname, num_threads=num_threads) |
| 51 | +# return video_reader |
| 52 | + |
| 53 | +# def video_loader(root, vid, ext, second, end_second, |
| 54 | +# chunk_len=300, fps=30, clip_length=32, |
| 55 | +# threads=1, |
| 56 | +# fast_rrc=False, rrc_params=(224, (0.5, 1.0)), |
| 57 | +# fast_rcc=False, rcc_params=(224, ), |
| 58 | +# jitter=False): |
| 59 | +# assert fps > 0, 'fps should be greater than 0' |
| 60 | + |
| 61 | +# if chunk_len == -1: |
| 62 | +# vr = get_video_reader( |
| 63 | +# osp.join(root, '{}.{}'.format(vid, ext)), |
| 64 | +# num_threads=threads, |
| 65 | +# fast_rrc=fast_rrc, rrc_params=rrc_params, |
| 66 | +# fast_rcc=fast_rcc, rcc_params=rcc_params, |
| 67 | +# ) |
| 68 | +# end_second = min(end_second, len(vr) / fps) |
| 69 | + |
| 70 | +# # calculate frame_ids |
| 71 | +# frame_offset = int(np.round(second * fps)) |
| 72 | +# total_duration = max(int((end_second - second) * fps), clip_length) |
| 73 | +# frame_ids = get_frame_ids(frame_offset, min(frame_offset + total_duration, len(vr)), num_segments=clip_length, jitter=jitter) |
| 74 | + |
| 75 | +# # load frames |
| 76 | +# assert max(frame_ids) < len(vr) |
| 77 | +# try: |
| 78 | +# frames = vr.get_batch(frame_ids).asnumpy() |
| 79 | +# except decord.DECORDError as error: |
| 80 | +# print(error) |
| 81 | +# frames = vr.get_batch([0] * len(frame_ids)).asnumpy() |
| 82 | + |
| 83 | +# return torch.from_numpy(frames.astype(np.float32)) |
| 84 | + |
| 85 | +# else: |
| 86 | +# chunk_start = int(second) // chunk_len * chunk_len |
| 87 | +# chunk_end = int(end_second) // chunk_len * chunk_len |
| 88 | +# while True: |
| 89 | +# video_filename = osp.join(root, '{}.{}'.format(vid, ext), '{}.{}'.format(chunk_end, ext)) |
| 90 | +# if not osp.exists(video_filename): |
| 91 | +# # print("{} does not exists!".format(video_filename)) |
| 92 | +# chunk_end -= chunk_len |
| 93 | +# else: |
| 94 | +# vr = decord.VideoReader(video_filename) |
| 95 | +# end_second = min(end_second, (len(vr) - 1) / fps + chunk_end) |
| 96 | +# assert chunk_start <= chunk_end |
| 97 | +# break |
| 98 | +# # calculate frame_ids |
| 99 | +# frame_ids = get_frame_ids( |
| 100 | +# int(np.round(second * fps)), |
| 101 | +# int(np.round(end_second * fps)), |
| 102 | +# num_segments=clip_length, jitter=jitter |
| 103 | +# ) |
| 104 | +# all_frames = [] |
| 105 | +# # allocate absolute frame-ids into the relative ones |
| 106 | +# for chunk in range(chunk_start, chunk_end + chunk_len, chunk_len): |
| 107 | +# rel_frame_ids = list(filter(lambda x: int(chunk * fps) <= x < int((chunk + chunk_len) * fps), frame_ids)) |
| 108 | +# rel_frame_ids = [int(frame_id - chunk * fps) for frame_id in rel_frame_ids] |
| 109 | +# vr = get_video_reader( |
| 110 | +# osp.join(root, '{}.{}'.format(vid, ext), '{}.{}'.format(chunk, ext)), |
| 111 | +# num_threads=threads, |
| 112 | +# fast_rrc=fast_rrc, rrc_params=rrc_params, |
| 113 | +# fast_rcc=fast_rcc, rcc_params=rcc_params, |
| 114 | +# ) |
| 115 | +# try: |
| 116 | +# frames = vr.get_batch(rel_frame_ids).asnumpy() |
| 117 | +# except decord.DECORDError as error: |
| 118 | +# print(error) |
| 119 | +# frames = vr.get_batch([0] * len(rel_frame_ids)).asnumpy() |
| 120 | +# except IndexError: |
| 121 | +# print(root, vid, ext, second, end_second) |
| 122 | +# all_frames.append(frames) |
| 123 | +# if sum(map(lambda x: x.shape[0], all_frames)) == clip_length: |
| 124 | +# break |
| 125 | +# res = torch.from_numpy(np.concatenate(all_frames, axis=0).astype(np.float32)) |
| 126 | +# assert res.shape[0] == clip_length, "{}, {}, {}, {}, {}, {}, {}".format(root, vid, second, end_second, res.shape[0], rel_frame_ids, frame_ids) |
| 127 | +# return res |
| 128 | + |
| 129 | +def process_EK100_video_with_decord(video_file, data_args, start_second, end_second, chunk_len): |
| 130 | + fps = 30 |
| 131 | + start_frame = int(start_second * fps) |
| 132 | + end_frame = int(end_second * fps) |
| 133 | + chunk_start = int(start_second) // chunk_len * chunk_len |
| 134 | + chunk_end = int(end_second) // chunk_len * chunk_len |
| 135 | + video_time = end_second - start_second |
| 136 | + while True: |
| 137 | + video_filename = os.path.join(video_file, '{}.MP4'.format(chunk_end)) |
| 138 | + if not os.path.exists(video_filename): |
| 139 | + # print("{} does not exists!".format(video_filename)) |
| 140 | + chunk_end -= chunk_len |
| 141 | + else: |
| 142 | + vr = VideoReader(video_filename, ctx=cpu(0), num_threads=1) |
| 143 | + end_second = min(end_second, (len(vr) - 1) / fps + chunk_end) |
| 144 | + assert chunk_start <= chunk_end |
| 145 | + break |
| 146 | + |
| 147 | + # calculate frame_ids |
| 148 | + frame_ids = get_frame_ids(start_frame, end_frame, num_segments=data_args.frames_upbound, jitter=False) |
| 149 | + frame_time = [i/fps for i in frame_ids] |
| 150 | + |
| 151 | + all_frames = [] |
| 152 | + # allocate absolute frame-ids into the relative ones |
| 153 | + for chunk in range(chunk_start, chunk_end + chunk_len, chunk_len): |
| 154 | + rel_frame_ids = list(filter(lambda x: int(chunk * fps) <= x < int((chunk + chunk_len) * fps), frame_ids)) |
| 155 | + rel_frame_ids = [int(frame_id - chunk * fps) for frame_id in rel_frame_ids] |
| 156 | + vr = VideoReader(os.path.join(video_file, '{}.MP4'.format(chunk)),ctx=cpu(0), num_threads=1) |
| 157 | + frames = vr.get_batch(rel_frame_ids).asnumpy() |
| 158 | + all_frames.append(frames) |
| 159 | + vr.seek(0) |
| 160 | + if sum(map(lambda x: x.shape[0], all_frames)) == data_args.frames_upbound: |
| 161 | + break |
| 162 | + |
| 163 | + video = np.concatenate(all_frames, axis=0).astype(np.float32) |
| 164 | + |
| 165 | + frame_time = ",".join([f"{i:.2f}s" for i in frame_time]) |
| 166 | + num_frames_to_sample = len(frame_ids) |
| 167 | + |
| 168 | + return video, video_time, frame_time, num_frames_to_sample |
| 169 | + |
25 | 170 | def process_video_with_decord(video_file, data_args): |
26 | 171 | vr = VideoReader(video_file, ctx=cpu(0), num_threads=1) |
27 | 172 | total_frame_num = len(vr) |
|
0 commit comments