diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 58244187..490a11ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,20 +32,20 @@ repos: args: ["check", "--select", "I", "--fix"] - id: ruff-format - - repo: local - hooks: - - id: no-underscore-md - name: "Disallow '_' in Markdown filenames" - language: system - entry: | - bash -c ' - # Report the offending files - echo "[pre-commit] ERROR: Found Markdown files with underscores:" >&2 - for file in "$@"; do - echo " - $file (use hyphens instead)" >&2 - done - exit 1 - ' - files: '.*\/[^\/]*_[^\/]*\.md$' - exclude: '^\.github/' - types: [file] + # - repo: local + # hooks: + # - id: no-underscore-md + # name: "Disallow '_' in Markdown filenames" + # language: system + # entry: | + # bash -c ' + # # Report the offending files + # echo "[pre-commit] ERROR: Found Markdown files with underscores:" >&2 + # for file in "$@"; do + # echo " - $file (use hyphens instead)" >&2 + # done + # exit 1 + # ' + # files: '.*\/[^\/]*_[^\/]*\.md$' + # exclude: '^\.github/' + # types: [file] diff --git a/README.md b/README.md index 2b6b659f..88ade2fb 100644 --- a/README.md +++ b/README.md @@ -1 +1,30 @@ -# NeMo VFM +# NeMo VFM: video foundation model collection + +NeMo VFM is a state-of-the-art framework for fast, large-scale training and inference of video world models. It unifies the latest diffusion-based and autoregressive techniques, prioritizing efficiency and performance from research prototyping to production deployment. + +## Projects + +This collection consists of 4 projects: +1. [Scalable diffusion training framework](nemo_vfm/diffusion/readme.rst) +2. [Accelerated diffusion world models](nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/README.md) +3. [Accelerated autoregressive world models](nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/README.md) +4. [Sparse attention for efficient diffusion inference](nemo_vfm/sparse_attention/README.md) + +## Citations + +If you find our code useful, please consider citing the following papers: +```bibtex +@article{patel2025training, + title={Training Video Foundation Models with NVIDIA NeMo}, + author={Patel, Zeeshan and He, Ethan and Mannan, Parth and Ren, Xiaowei and Wolf, Ryan and Agarwal, Niket and Huffman, Jacob and Wang, Zhuoyao and Wang, Carl and Chang, Jack and others}, + journal={arXiv preprint arXiv:2503.12964}, + year={2025} +} + +@article{agarwal2025cosmos, + title={Cosmos world foundation model platform for physical ai}, + author={Agarwal, Niket and Ali, Arslan and Bala, Maciej and Balaji, Yogesh and Barker, Erik and Cai, Tiffany and Chattopadhyay, Prithvijit and Chen, Yongxin and Cui, Yin and Ding, Yifan and others}, + journal={arXiv preprint arXiv:2501.03575}, + year={2025} +} +``` \ No newline at end of file diff --git a/nemo_vfm/diffusion/Dockerfile b/nemo_vfm/diffusion/Dockerfile new file mode 100644 index 00000000..8e555b33 --- /dev/null +++ b/nemo_vfm/diffusion/Dockerfile @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvcr.io/nvidia/nemo:24.07 + +RUN pip install --no-cache-dir diffusers==0.30.3 git+https://github.com/NVIDIA/NeMo-Run.git av megatron-energon ffmpeg-python==0.2.0 imageio-ffmpeg==0.4.8 imageio==2.26.0 opencv-python==4.8.0.74 opencv-python-headless==4.8.0.74 mediapy diff --git a/nemo_vfm/diffusion/__init__.py b/nemo_vfm/diffusion/__init__.py new file mode 100644 index 00000000..9e325007 --- /dev/null +++ b/nemo_vfm/diffusion/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/assets/mixed_training.png b/nemo_vfm/diffusion/assets/mixed_training.png new file mode 100644 index 00000000..2226e4c4 Binary files /dev/null and b/nemo_vfm/diffusion/assets/mixed_training.png differ diff --git a/nemo_vfm/diffusion/assets/pipeline_conditioning.png b/nemo_vfm/diffusion/assets/pipeline_conditioning.png new file mode 100644 index 00000000..0856489a Binary files /dev/null and b/nemo_vfm/diffusion/assets/pipeline_conditioning.png differ diff --git a/nemo_vfm/diffusion/assets/st_dit_hybrid_parallel.png b/nemo_vfm/diffusion/assets/st_dit_hybrid_parallel.png new file mode 100644 index 00000000..dcbe19fa Binary files /dev/null and b/nemo_vfm/diffusion/assets/st_dit_hybrid_parallel.png differ diff --git a/nemo_vfm/diffusion/data/__init__.py b/nemo_vfm/diffusion/data/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/data/camera.py b/nemo_vfm/diffusion/data/camera.py new file mode 100644 index 00000000..3297ddc4 --- /dev/null +++ b/nemo_vfm/diffusion/data/camera.py @@ -0,0 +1,639 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import numpy as np +import torch + + +class Pose: + """ + A class of operations on camera poses (PyTorch tensors with shape [...,3,4]). + Each [3,4] camera pose takes the form of [R|t]. + """ + + def __call__(self, R=None, t=None): + # Construct a camera pose from the given R and/or t. + assert R is not None or t is not None + if R is None: + if not isinstance(t, torch.Tensor): + t = torch.tensor(t) + R = torch.eye(3, device=t.device).repeat(*t.shape[:-1], 1, 1) + elif t is None: + if not isinstance(R, torch.Tensor): + R = torch.tensor(R) + t = torch.zeros(R.shape[:-1], device=R.device) + else: + if not isinstance(R, torch.Tensor): + R = torch.tensor(R) + if not isinstance(t, torch.Tensor): + t = torch.tensor(t) + assert R.shape[:-1] == t.shape and R.shape[-2:] == (3, 3) + R = R.float() + t = t.float() + pose = torch.cat([R, t[..., None]], dim=-1) # [...,3,4] + assert pose.shape[-2:] == (3, 4) + return pose + + def invert(self, pose, use_inverse=False): + # Invert a camera pose. + R, t = pose[..., :3], pose[..., 3:] + R_inv = R.inverse() if use_inverse else R.transpose(-1, -2) + t_inv = (-R_inv @ t)[..., 0] + pose_inv = self(R=R_inv, t=t_inv) + return pose_inv + + def compose(self, pose_list): + # Compose a sequence of poses together. + # pose_new(x) = poseN o ... o pose2 o pose1(x) + pose_new = pose_list[0] + for pose in pose_list[1:]: + pose_new = self.compose_pair(pose_new, pose) + return pose_new + + def compose_pair(self, pose_a, pose_b): + # pose_new(x) = pose_b o pose_a(x) + R_a, t_a = pose_a[..., :3], pose_a[..., 3:] + R_b, t_b = pose_b[..., :3], pose_b[..., 3:] + R_new = R_b @ R_a + t_new = (R_b @ t_a + t_b)[..., 0] + pose_new = self(R=R_new, t=t_new) + return pose_new + + def scale_center(self, pose, scale): + """Scale the camera center from the origin. + 0 = R@c+t --> c = -R^T@t (camera center in world coordinates) + 0 = R@(sc)+t' --> t' = -R@(sc) = -R@(-R^T@st) = st + """ + R, t = pose[..., :3], pose[..., 3:] + pose_new = torch.cat([R, t * scale], dim=-1) + return pose_new + + def interpolate(self, pose_a, pose_b, alpha): + """Interpolate between two poses with Slerp. + Args: + pose_a (tensor [...,3,4]): Pose at time t=0. + pose_b (tensor [...,3,4]): Pose at time t=1. + alpha (tensor [...,1]): Interpolation parameter. + Returns: + pose (tensor [...,3,4]): Pose at time t. + """ + R_a, t_a = pose_a[..., :3], pose_a[..., 3:] + R_b, t_b = pose_b[..., :3], pose_b[..., 3:] + q_a = quaternion.R_to_q(R_a) # [...,4] + q_b = quaternion.R_to_q(R_b) # [...,4] + q_intp = quaternion.interpolate(q_a, q_b, alpha) # [...,4] + R_intp = quaternion.q_to_R(q_intp) # [...,3,3] + t_intp = (1 - alpha) * t_a + alpha * t_b # [...,3] + pose_intp = torch.cat([R_intp, t_intp], dim=-1) # [...,3,4] + return pose_intp + + def to_4x4(self, pose): + last_row = torch.tensor([0, 0, 0, 1], device=pose.device)[None, None].expand(pose.shape[0], 1, 4) + return torch.cat([pose, last_row], dim=-2) + + +class Lie: + """ + Lie algebra for SO(3) and SE(3) operations in PyTorch. + """ + + def so3_to_SO3(self, w): # [..., 3] + wx = self.skew_symmetric(w) + theta = w.norm(dim=-1)[..., None, None] + eye = torch.eye(3, device=w.device, dtype=torch.float32) + A = self.taylor_A(theta) + B = self.taylor_B(theta) + R = eye + A * wx + B * wx @ wx + return R + + def SO3_to_so3(self, R, eps=1e-7): # [..., 3, 3] + trace = R[..., 0, 0] + R[..., 1, 1] + R[..., 2, 2] + theta = ((trace - 1) / 2).clamp(-1 + eps, 1 - eps).acos_()[ + ..., None, None + ] % np.pi # ln(R) will explode if theta==pi + lnR = 1 / (2 * self.taylor_A(theta) + 1e-8) * (R - R.transpose(-2, -1)) # FIXME: wei-chiu finds it weird + w0, w1, w2 = lnR[..., 2, 1], lnR[..., 0, 2], lnR[..., 1, 0] + w = torch.stack([w0, w1, w2], dim=-1) + return w + + def se3_to_SE3(self, wu): # [...,3] + w, u = wu.split([3, 3], dim=-1) + wx = self.skew_symmetric(w) + theta = w.norm(dim=-1)[..., None, None] + eye = torch.eye(3, device=w.device, dtype=torch.float32) + A = self.taylor_A(theta) + B = self.taylor_B(theta) + C = self.taylor_C(theta) + R = eye + A * wx + B * wx @ wx + V = eye + B * wx + C * wx @ wx + Rt = torch.cat([R, (V @ u[..., None])], dim=-1) + return Rt + + def SE3_to_se3(self, Rt, eps=1e-8): # [...,3,4] + R, t = Rt.split([3, 1], dim=-1) + w = self.SO3_to_so3(R) + wx = self.skew_symmetric(w) + theta = w.norm(dim=-1)[..., None, None] + eye = torch.eye(3, device=w.device, dtype=torch.float32) + A = self.taylor_A(theta) + B = self.taylor_B(theta) + invV = eye - 0.5 * wx + (1 - A / (2 * B)) / (theta**2 + eps) * wx @ wx + u = (invV @ t)[..., 0] + wu = torch.cat([w, u], dim=-1) + return wu + + def skew_symmetric(self, w): + w0, w1, w2 = w.unbind(dim=-1) + zero = torch.zeros_like(w0) + wx = torch.stack( + [ + torch.stack([zero, -w2, w1], dim=-1), + torch.stack([w2, zero, -w0], dim=-1), + torch.stack([-w1, w0, zero], dim=-1), + ], + dim=-2, + ) + return wx + + def taylor_A(self, x, nth=10): + # Taylor expansion of sin(x)/x. + ans = torch.zeros_like(x) + denom = 1.0 + for i in range(nth + 1): + if i > 0: + denom *= (2 * i) * (2 * i + 1) + ans = ans + (-1) ** i * x ** (2 * i) / denom + return ans + + def taylor_B(self, x, nth=10): + # Taylor expansion of (1-cos(x))/x**2. + ans = torch.zeros_like(x) + denom = 1.0 + for i in range(nth + 1): + denom *= (2 * i + 1) * (2 * i + 2) + ans = ans + (-1) ** i * x ** (2 * i) / denom + return ans + + def taylor_C(self, x, nth=10): + # Taylor expansion of (x-sin(x))/x**3. + ans = torch.zeros_like(x) + denom = 1.0 + for i in range(nth + 1): + denom *= (2 * i + 2) * (2 * i + 3) + ans = ans + (-1) ** i * x ** (2 * i) / denom + return ans + + +class Quaternion: + def q_to_R(self, q): # [...,4] + # https://en.wikipedia.org/wiki/Rotation_matrix#Quaternion + qa, qb, qc, qd = q.unbind(dim=-1) + R = torch.stack( + [ + torch.stack([1 - 2 * (qc**2 + qd**2), 2 * (qb * qc - qa * qd), 2 * (qa * qc + qb * qd)], dim=-1), + torch.stack([2 * (qb * qc + qa * qd), 1 - 2 * (qb**2 + qd**2), 2 * (qc * qd - qa * qb)], dim=-1), + torch.stack([2 * (qb * qd - qa * qc), 2 * (qa * qb + qc * qd), 1 - 2 * (qb**2 + qc**2)], dim=-1), + ], + dim=-2, + ) + return R + + def R_to_q(self, R, eps=1e-6): # [...,3,3] + # https://en.wikipedia.org/wiki/Rotation_matrix#Quaternion + row0, row1, row2 = R.unbind(dim=-2) + R00, R01, R02 = row0.unbind(dim=-1) + R10, R11, R12 = row1.unbind(dim=-1) + R20, R21, R22 = row2.unbind(dim=-1) + t = R[..., 0, 0] + R[..., 1, 1] + R[..., 2, 2] + r = (1 + t + eps).sqrt() + qa = 0.5 * r + qb = (R21 - R12).sign() * 0.5 * (1 + R00 - R11 - R22 + eps).sqrt() + qc = (R02 - R20).sign() * 0.5 * (1 - R00 + R11 - R22 + eps).sqrt() + qd = (R10 - R01).sign() * 0.5 * (1 - R00 - R11 + R22 + eps).sqrt() + q = torch.stack([qa, qb, qc, qd], dim=-1) + return q + + def invert(self, q): # [...,4] + qa, qb, qc, qd = q.unbind(dim=-1) + norm = q.norm(dim=-1, keepdim=True) + q_inv = torch.stack([qa, -qb, -qc, -qd], dim=-1) / norm**2 + return q_inv + + def product(self, q1, q2): # [...,4] + q1a, q1b, q1c, q1d = q1.unbind(dim=-1) + q2a, q2b, q2c, q2d = q2.unbind(dim=-1) + hamil_prod = torch.stack( + [ + q1a * q2a - q1b * q2b - q1c * q2c - q1d * q2d, + q1a * q2b + q1b * q2a + q1c * q2d - q1d * q2c, + q1a * q2c - q1b * q2d + q1c * q2a + q1d * q2b, + q1a * q2d + q1b * q2c - q1c * q2b + q1d * q2a, + ], + dim=-1, + ) + return hamil_prod + + def interpolate(self, q1, q2, alpha): # [...,4],[...,4],[...,1] + # https://en.wikipedia.org/wiki/Slerp + cos_angle = (q1 * q2).sum(dim=-1, keepdim=True) # [...,1] + flip = cos_angle < 0 + q1 = q1 * (~flip) - q1 * flip # [...,4] + theta = cos_angle.abs().acos() # [...,1] + slerp = (((1 - alpha) * theta).sin() * q1 + (alpha * theta).sin() * q2) / theta.sin() # [...,4] + return slerp + + +pose = Pose() +lie = Lie() +quaternion = Quaternion() + + +def to_hom(X): + # Get homogeneous coordinates of the input. + X_hom = torch.cat([X, torch.ones_like(X[..., :1])], dim=-1) + return X_hom + + +# Basic operations of transforming 3D points between world/camera/image coordinates. +def world2cam(X, pose): # [B,N,3] + X_hom = to_hom(X) + return X_hom @ pose.transpose(-1, -2) + + +def cam2img(X, cam_intr): + return X @ cam_intr.transpose(-1, -2) + + +def img2cam(X, cam_intr): + _dtype = cam_intr.dtype + X = X.float() + cam_intr = cam_intr.float() + result = X @ cam_intr.inverse().transpose(-1, -2) + return result.to(dtype=_dtype) + + +def cam2world(X, pose): + _dtype = pose.dtype + X = X.float() + pose = pose.float() + X_hom = to_hom(X) + pose_inv = Pose().invert(pose) + result = X_hom @ pose_inv.transpose(-1, -2) + return result.to(dtype=_dtype) + + +def angle_to_rotation_matrix(a, axis): + # Get the rotation matrix from Euler angle around specific axis. + roll = dict(X=1, Y=2, Z=0)[axis] + if isinstance(a, float): + a = torch.tensor(a) + zero = torch.zeros_like(a) + eye = torch.ones_like(a) + M = torch.stack( + [ + torch.stack([a.cos(), -a.sin(), zero], dim=-1), + torch.stack([a.sin(), a.cos(), zero], dim=-1), + torch.stack([zero, zero, eye], dim=-1), + ], + dim=-2, + ) + M = M.roll((roll, roll), dims=(-2, -1)) + return M + + +def get_center_and_ray(pose, intr, image_size): + """ + Args: + pose (tensor [3,4]/[B,3,4]): Camera pose. + intr (tensor [3,3]/[B,3,3]): Camera intrinsics. + image_size (list of int): Image size. + Returns: + center_3D (tensor [HW,3]/[B,HW,3]): Center of the camera. + ray (tensor [HW,3]/[B,HW,3]): Ray of the camera with depth=1 (note: not unit ray). + """ + assert pose.dtype == torch.float32 and intr.dtype == torch.float32, ( + f"pose and intr should be float32, got {pose.dtype} and {intr.dtype}" + ) + + H, W = image_size + # Given the intrinsic/extrinsic matrices, get the camera center and ray directions. + with torch.no_grad(): + # Compute image coordinate grid. + X, Y = get_pixel_grid(W, H, pose.device, normalized_coordinate=False) # [H,W] + xy_grid = torch.stack([X, Y], dim=-1).view(-1, 2) # [HW,2] + # Compute center and ray. + if len(pose.shape) == 3: + batch_size = len(pose) + xy_grid = xy_grid.repeat(batch_size, 1, 1) # [B,HW,2] + grid_3D = img2cam(to_hom(xy_grid), intr) # [HW,3]/[B,HW,3] + center_3D = torch.zeros_like(grid_3D) # [HW,3]/[B,HW,3] + # Transform from camera to world coordinates. + grid_3D = cam2world(grid_3D, pose) # [HW,3]/[B,HW,3] + center_3D = cam2world(center_3D, pose) # [HW,3]/[B,HW,3] + ray = grid_3D - center_3D # [B,HW,3] + return center_3D, ray + + +def get_pixel_grid(width: int, height: int, device: torch.device, normalized_coordinate: bool = False): + """Generate pixel grid given the image size. + + Args: + width (int): image width + height (int): image height + device (torch.device) + normalized_coordinate (bool, optional): normalized coordinate is between 0 and 1. Defaults to False. + + Returns: + torch.tensor: x,y pixel grid + """ + y_range = torch.arange(height, dtype=torch.float32, device=device).add_(0.5) + x_range = torch.arange(width, dtype=torch.float32, device=device).add_(0.5) + if normalized_coordinate: + y_range = y_range / height + x_range = x_range / width + y, x = torch.meshgrid(y_range, x_range, indexing="ij") # [H, W] + return x, y + + +def get_3D_points_from_dist( + center: torch.tensor, ray_unit: torch.tensor, dist: torch.tensor, multiple_samples_per_ray: bool = False +): + """Convert dist to 3D points in the world coordinate. + + Args: + center (torch.tensor): camer center in world coordinates, [..., 3] + ray_unit (torch.tensor): ray directions (unit vector), [..., 3] + dist (torch.tensor): distance along the ray, [..., 1] or [..., N_samples, 1] + if sampling muliple points along rays + multiple_samples_per_ray (bool): If True, dist is [..., N_samples, 1] + + Returns: + torch.tensor: [..., 3] or [..., N_samples, 3] + """ + assert torch.allclose(ray_unit.norm(dim=-1), torch.ones_like(ray_unit.norm(dim=-1))), ( + f"ray_unit norm is not equal to 1, max {ray_unit.norm(dim=-1).max()} min {ray_unit.norm(dim=-1).min()}" + ) + if multiple_samples_per_ray: + assert len(dist.shape) == len(center.shape) + 1 + center, ray_unit = center[..., None, :], ray_unit[..., None, :] # [...,1,3] + else: + assert len(dist.shape) == len(center.shape), f"dist shape {dist.shape} center shape {center.shape}" + points_3D = center + ray_unit * dist # [...,3]/[...,N_samples,3] + return points_3D + + +def get_3D_points_from_depth( + center: torch.tensor, ray: torch.tensor, depth: torch.tensor, multiple_samples_per_ray: bool = False +): + """Convert depth to 3D points in the world coordinate. + NOTE: this function assuems the ray is NOT noramlized and returned directly from get_center_and_ray()!! + + Args: + center (torch.tensor): camer center in world coordinates, [..., 3] + ray (torch.tensor): ray directions (z component is 1), [..., 3] + depth (torch.tensor): z depth from camera center, [..., 1] or [..., N_samples, 1] + if sampling muliple points along rays + multiple_samples_per_ray (bool): If True, depth is [..., N_samples, 1] + + Returns: + torch.tensor: [..., 3] or [..., N_samples, 3] + """ + if multiple_samples_per_ray: + assert len(depth.shape) == len(center.shape) + 1 + center, ray = center[..., None, :], ray[..., None, :] # [...,1,3] + else: + assert len(depth.shape) == len(center.shape) + points_3D = center + ray * depth # [...,3]/[...,N,3] + return points_3D + + +def convert_NDC(center, ray, intr, near=1): + # Shift camera center (ray origins) to near plane (z=1). + # (Unlike conventional NDC, we assume the cameras are facing towards the +z direction.) + center = center + (near - center[..., 2:]) / ray[..., 2:] * ray + # Projection. + cx, cy, cz = center.unbind(dim=-1) # [...,R] + rx, ry, rz = ray.unbind(dim=-1) # [...,R] + scale_x = intr[..., 0, 0] / intr[..., 0, 2] # [...] + scale_y = intr[..., 1, 1] / intr[..., 1, 2] # [...] + cnx = scale_x[..., None] * (cx / cz) + cny = scale_y[..., None] * (cy / cz) + cnz = 1 - 2 * near / cz + rnx = scale_x[..., None] * (rx / rz - cx / cz) + rny = scale_y[..., None] * (ry / rz - cy / cz) + rnz = 2 * near / cz + center_ndc = torch.stack([cnx, cny, cnz], dim=-1) # [...,R,3] + ray_ndc = torch.stack([rnx, rny, rnz], dim=-1) # [...,R,3] + return center_ndc, ray_ndc + + +def convert_NDC2(center, ray, intr): + # Similar to convert_NDC() but shift the ray origins to its own image plane instead of the global near plane. + # Also this version is much more interpretable. + scale_x = intr[..., 0, 0] / intr[..., 0, 2] # [...] + scale_y = intr[..., 1, 1] / intr[..., 1, 2] # [...] + # Get the metric image plane (i.e. new "center"): (sx*cx/cz, sy*cy/cz, 1-2/cz). + center = center + ray # This is the key difference. + cx, cy, cz = center.unbind(dim=-1) # [...,R] + image_plane = torch.stack([scale_x[..., None] * cx / cz, scale_x[..., None] * cy / cz, 1 - 2 / cz], dim=-1) + # Get the infinity plane: (sx*rx/rz, sy*ry/rz, 1). + rx, ry, rz = ray.unbind(dim=-1) # [...,R] + inf_plane = torch.stack([scale_x[..., None] * rx / rz, scale_y[..., None] * ry / rz, torch.ones_like(rz)], dim=-1) + # The NDC ray is the difference between the two planes, assuming t \in [0,1]. + ndc_ray = inf_plane - image_plane + return image_plane, ndc_ray + + +def rotation_distance(R1, R2, eps=1e-7): + # http://www.boris-belousov.net/2016/12/01/quat-dist/ + R_diff = R1 @ R2.transpose(-2, -1) + trace = R_diff[..., 0, 0] + R_diff[..., 1, 1] + R_diff[..., 2, 2] + angle = ((trace - 1) / 2).clamp(-1 + eps, 1 - eps).acos_() # numerical stability near -1/+1 + return angle + + +def get_oscil_novel_view_poses(N=60, angle=0.05, dist=5): + # Create circular viewpoints (small oscillations). + theta = torch.arange(N) / N * 2 * np.pi + R_x = angle_to_rotation_matrix((theta.sin() * angle).asin(), "X") + R_y = angle_to_rotation_matrix((theta.cos() * angle).asin(), "Y") + pose_rot = pose(R=R_y @ R_x) + pose_shift = pose(t=[0, 0, dist]) + pose_oscil = pose.compose([pose.invert(pose_shift), pose_rot, pose_shift]) + return pose_oscil + + +def cross_product_matrix(x): + """Matrix form of cross product opertaion. + + param x: [3,] tensor. + return: [3, 3] tensor representing the matrix form of cross product. + """ + return torch.tensor( + [ + [0, -x[2], x[1]], + [x[2], 0, -x[0]], + [ + -x[1], + x[0], + 0, + ], + ] + ) + + +def essential_matrix(poses): + """Compute Essential Matrix from a relative pose. + + param poses: [views, 3, 4] tensor representing relative poses. + return: [views, 3, 3] tensor representing Essential Matrix. + """ + r = poses[..., 0:3] + t = poses[..., 3] + tx = torch.stack([cross_product_matrix(tt) for tt in t], axis=0) + return tx @ r + + +def fundamental_matrix(poses, intr1, intr2): + """Compute Fundamental Matrix from a relative pose and intrinsics. + + param poses: [views, 3, 4] tensor representing relative poses. + intr1: [3, 3] tensor. Camera intrinsic of reference image. + intr2: [views, 3, 3] tensor. Camera Intrinsic of target image. + return: [views, 3, 3] tensor representing Fundamental Matrix. + """ + return intr2.inverse().transpose(-1, -2) @ essential_matrix(poses) @ intr1.inverse() + + +def get_ray_depth_plane_intersection(center, ray, depths): + """Compute the intersection of a ray with a depth plane. + Args: + center (tensor [B,HW,3]): Camera center of the target pose. + ray (tensor [B,HW,3]): Ray direction of the target pose. + depth (tensor [L]): The depth values from the source view (e.g. for MPI planes). + Returns: + intsc_points (tensor [B,HW,L,3]): Intersecting 3D points with the MPI. + """ + # Each 3D point x along the ray v from center c can be written as x = c+t*v. + # Plane equation: n@x = d, where normal n = (0,0,1), d = depth. + # --> t = (d-n@c)/(n@v). + # --> x = c+t*v = c+(d-n@c)/(n@v)*v. + center, ray = center[:, :, None], ray[:, :, None] # [B,HW,L,3], [B,HW,1,3] + depths = depths[None, None, :, None] # [1,1,L,1] + intsc_points = center + (depths - center[..., 2:]) / ray[..., 2:] * ray # [B,HW,L,3] + return intsc_points + + +def unit_view_vector_to_rotation_matrix(v, axes="ZYZ"): + """ + Args: + v (tensor [...,3]): Unit vectors on the view sphere. + axes: rotation axis order. + + Returns: + rotation_matrix (tensor [...,3,3]): rotation matrix R @ v + [0, 0, 1] = 0. + """ + alpha = torch.arctan2(v[..., 1], v[..., 0]) # [...] + beta = np.pi - v[..., 2].arccos() # [...] + euler_angles = torch.stack([torch.ones_like(alpha) * np.pi / 2, -beta, alpha], dim=-1) # [...,3] + rot2 = angle_to_rotation_matrix(euler_angles[..., 2], axes[2]) # [...,3,3] + rot1 = angle_to_rotation_matrix(euler_angles[..., 1], axes[1]) # [...,3,3] + rot0 = angle_to_rotation_matrix(euler_angles[..., 0], axes[0]) # [...,3,3] + rot = rot2 @ rot1 @ rot0 # [...,3,3] + return rot.transpose(-2, -1) + + +def sample_on_spherical_cap(anchor, N, max_angle, min_angle=0.0): + """Sample n points on the view hemisphere within the angle to x. + Args: + anchor (tensor [...,3]): Reference 3-D unit vector on the view hemisphere. + N (int): Number of sampled points. + max_angle (float): Sampled points should have max angle to x. + Returns: + sampled_points (tensor [...,N,3]): Sampled points on the spherical caps. + """ + batch_shape = anchor.shape[:-1] + # First, sample uniformly on a unit 2D disk. + radius = torch.rand(*batch_shape, N, device=anchor.device) # [...,N] + h_max = 1 - np.cos(max_angle) # spherical cap height + h_min = 1 - np.cos(min_angle) # spherical cap height + radius = (radius * (h_max - h_min) + h_min) / h_max + theta = torch.rand(*batch_shape, N, device=anchor.device) * 2 * np.pi # [...,N] + x = radius.sqrt() * theta.cos() # [...,N] + y = radius.sqrt() * theta.sin() # [...,N] + # Reparametrize to a unit spherical cap with height h. + # http://marc-b-reynolds.github.io/distribution/2016/11/28/Uniform.html + k = h_max * radius # [...,N] + s = (h_max * (2 - k)).sqrt() # [...,N] + points = torch.stack([s * x, s * y, 1 - k], dim=-1) # [...,N,3] + # Transform to center around the anchor. + ref_z = torch.tensor([0.0, 0.0, 1.0], device=anchor.device) + v = -anchor.cross(ref_z) # [...,3] + ss_v = lie.skew_symmetric(v) # [...,3,3] + R = torch.eye(3, device=anchor.device) + ss_v + ss_v @ ss_v / (1 + anchor @ ref_z)[..., None, None] # [...,3,3] + points = points @ R.transpose(-2, -1) # [...,N,3] + return points + + +def sample_on_spherical_cap_northern(anchor, N, max_angle, away_from=None, max_reject_count=None): + """Sample n points only the northern view hemisphere within the angle to x.""" + + def find_invalid_points(points): + southern = points[..., 2] < 0 # [...,N] + if away_from is not None: + cosine_ab = (away_from * anchor).sum(dim=-1, keepdim=True) # [...,1] + cosine_ac = (away_from[..., None, :] * points).sum(dim=-1) # [...,N] + not_outwards = cosine_ab < cosine_ac # [...,N] + invalid = southern | not_outwards + else: + invalid = southern + return invalid + + assert (anchor[..., 2] > 0).all() + assert anchor.norm(dim=-1).allclose(torch.ones_like(anchor[..., 0])) + points = sample_on_spherical_cap(anchor, N, max_angle) # [...,N,3] + invalid = find_invalid_points(points) + count = 0 + while invalid.any(): + # Reject and resample. + points_resample = sample_on_spherical_cap(anchor, N, max_angle) + points[invalid] = points_resample[invalid] + invalid = find_invalid_points(points) + count += 1 + if max_reject_count and count > max_reject_count: + points = anchor.repeat(N, 1) + return points + + +def depth_to_pointcloud(depth: torch.tensor, intr: torch.tensor, extr: torch.tensor): + """Convert depth to pointcloud. + Args: + depth (torch.tensor): [1,H,W]/[B,1,H,W] + intr (torch.tensor): [3,3]/[B,3,3] + extr (torch.tensor): [3,4]/[B,3,4] + + Returns: + pc (torch.tensor): [HW,3] + """ + + assert len(depth.shape) == len(intr.shape) + 1, ( + f"dist ({depth.shape}) and intr ({intr.shape}) should have the same batch size" + ) + # convert depth to pointcloud + center, ray = get_center_and_ray(extr, intr, depth.shape[-2:]) + depth = depth.view(*center.shape[:-1], 1) # [HW, 1]/[B,HW,1] + pc = get_3D_points_from_depth(center, ray, depth) + return pc # HW,3/B,HW,3 diff --git a/nemo_vfm/diffusion/data/camera_ctrl_utils.py b/nemo_vfm/diffusion/data/camera_ctrl_utils.py new file mode 100644 index 00000000..78fa0896 --- /dev/null +++ b/nemo_vfm/diffusion/data/camera_ctrl_utils.py @@ -0,0 +1,159 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import numpy as np +import torch +from nemo.collections.diffusion.data import camera +from nemo.collections.diffusion.data.camera import get_center_and_ray + + +def plucker_coordinates(pose: torch.tensor, intr: torch.tensor, width: int, height: int): + """Return plücker coordinates from pose and intrinsics. Plücker coordinates are defined as + [(rx,ry,rz),(rx,ry,rz)x(cx,cy,cz)] where (cx,cy,cz) is the camera origin + and (rx,ry,rz) is the direction of the ray. + Plücker coordinates are used to represent a line in 3D space. + + Useful references: + - https://www.euclideanspace.com/maths/geometry/elements/line/plucker/index.htm + + + Args: + pose (torch.tensor): Extrinsics [B,3,4] + intr (torch.tensor): Intrinsics [B,3,3] + width (int): Image width + height (int): Image height + + Returns: + torch.tensor: plücker coordinates + """ + center, ray = get_center_and_ray(pose, intr, [height, width]) # [B,HW,3] + ray = ray / torch.norm(ray, dim=-1, keepdim=True) # [B,HW,3], unit length + plucker_coords = torch.cat([torch.cross(center, ray, dim=-1), ray], dim=-1) # [B,HW,6] + return plucker_coords + + +def get_relative_pose(pose_list: list[torch.Tensor | np.ndarray]) -> list[np.ndarray]: + """ + Convert a list of 3x4 world to camera pose to relative pose to the first frame + Args: + pose_list (list[torch.Tensor | np.ndarray]): List of 3x4 world to camera pose + Returns: + ret_poses (list[np.ndarray]): List of relative poses + """ + if isinstance(pose_list[0], np.ndarray): + poses = torch.from_numpy(np.stack(list(pose_list), axis=0)) # [N,3,4] + else: + poses = torch.stack(list(pose_list), dim=0) # [N,3,4] + pose_0 = poses[:1] + pose_0_inv = camera.pose.invert(pose_0) + rel_poses = camera.pose.compose_pair(pose_0_inv, poses) + # Homogeneous form (4x4) + rel_poses_4x4 = torch.eye(4).repeat(len(rel_poses), 1, 1) + rel_poses_4x4[:, :3, :] = rel_poses + return rel_poses_4x4.numpy() + + +def estimate_pose_list_to_plucker_embedding( + pose_list: list, + latent_compression_ratio_h: int, + latent_compression_ratio_w: int, + image_size: torch.tensor, + use_relative_pose: bool = True, +) -> torch.tensor: + """ + Convert a list of pose to plücker coordinates + Args: + pose_list (list): List of pose, each element is a dict with keys "intrinsics", "rotation", "translation" + e.g. {'intrinsics': [[0.4558800160884857, 0.0, 0.5], [0.0, 0.8124798536300659, 0.5], [0.0, 0.0, 0.0]], + 'rotation': [[0.5067835450172424, 0.4129045605659485, -0.7567564249038696], + [-0.41741496324539185, 0.8855977654457092, 0.20366966724395752], + [0.7542779445648193, 0.21266502141952515, 0.6211589574813843] + ], + 'translation': [1.5927585363388062, -0.41845059394836426, 0.6559827327728271]} + image_size (torch.tensor): Image size of the current video after processing, the input is + h_after_padded, w_after_padded, h_after_resize, w_after_resize, + e.g. [ 704., 1280., 704., 1252.] for input with raw shape [720, 1280] + latent_compression_ratio_h (int): compression height of the plücker embedding image + latent_compression_ratio_w (int): compression width of the plücker embedding image + use_relative_pose (bool): Whether to use relative pose + Returns: + plücker_coords (torch.tensor): Plücker embedding of shape [num_frame, HW, 6] + """ + num_frame = len(pose_list) + # e.g. 704, 1280, 704, 1252 + h_after_padded, w_after_padded, h_after_resize, w_after_resize = image_size + H = h_after_padded.item() // latent_compression_ratio_h # e.g. 704 / 8 = 88 + W = w_after_padded.item() // latent_compression_ratio_w # e.g. 1280 / 8 = 160 + ratio_w = w_after_resize.item() / w_after_padded.item() + ratio_h = h_after_resize.item() / h_after_padded.item() + + H = int(H) + W = int(W) + # Compute mv_intr_denormalized + mv_intr_denormalized = [] + for p in pose_list: + intrinsic = torch.tensor(p["intrinsics"]) + intrinsic[2, 2] = 1 + intrinsic[0, :] *= W * ratio_w + intrinsic[1, :] *= H * ratio_h + mv_intr_denormalized.append(intrinsic) + + mv_pose = [ + torch.cat([torch.tensor(p["rotation"]), torch.tensor(p["translation"]).unsqueeze(1)], dim=1) for p in pose_list + ] + + # Convert to pose relative to the first frame + if use_relative_pose: + mv_pose = get_relative_pose(mv_pose) + mv_intr_denormalized = torch.stack(mv_intr_denormalized) + mv_pose = torch.tensor(np.stack(mv_pose)) + mv_pose = mv_pose[:, :3] # B*N,3,4 + mv_intr_denormalized = mv_intr_denormalized.view(num_frame, 3, 3) # B*N,3,3 + + # plucker coordinates to encode pose + plucker_coords = plucker_coordinates(mv_pose, mv_intr_denormalized, W, H) # [B,HW,6] + + return plucker_coords, H, W + + +def normalize_camera_trajectory_to_unit_sphere(pose_list: list[dict]) -> None: + """ + Normalize the camera trajectory to fit within a unit sphere. + This function takes a list of camera poses, each represented as a dictionary with a "translation" key, + and normalizes the translation vectors such that the maximum distance between any two cameras is 1. + The normalization is done in-place. + Args: + pose_list (list[dict]): A list of dictionaries, where each dictionary contains a "translation" key + with a list or array of three floats representing the camera translation vector. + Returns: + None + """ + translation = np.array([pose["translation"] for pose in pose_list]) # [N,3] + + # Find the max distance between any two cameras. It is equivalent to the max distance of translation vectors. + def _longest_distance(points): + # Compute the pairwise distances. + diff = points[:, None, :] - points[None, :, :] + distances = np.linalg.norm(diff, axis=-1) + # Find the maximum distance + max_distance = np.max(distances) + return max_distance + + max_distance = _longest_distance(translation) + for pose in pose_list: + trans = np.array(pose["translation"]) + trans /= max_distance + pose["translation"] = trans.tolist() diff --git a/nemo_vfm/diffusion/data/diffusion_energon_datamodule.py b/nemo_vfm/diffusion/data/diffusion_energon_datamodule.py new file mode 100644 index 00000000..f3cebde9 --- /dev/null +++ b/nemo_vfm/diffusion/data/diffusion_energon_datamodule.py @@ -0,0 +1,147 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import logging +from typing import Any, Dict, Literal + +from megatron.energon import DefaultTaskEncoder, get_train_dataset +from nemo.collections.multimodal.data.energon.base import EnergonMultiModalDataModule +from pytorch_lightning.utilities.types import EVAL_DATALOADERS + + +class DiffusionDataModule(EnergonMultiModalDataModule): + """ + A PyTorch Lightning DataModule for handling multimodal datasets with images and text. + + This data module is designed to work with multimodal datasets that involve both images and text. + It provides a seamless interface to load training and validation data, manage batching, and handle + the state of the data pipeline across training epochs. The module integrates with the Megatron-Energon + framework for efficient data handling in large-scale distributed training. + + Attributes: + path (str): Path to the energon dataset. + tokenizer (Tokenizer): The tokenizer used for processing text. + image_processor (ImageProcessor): The image processor used for preprocessing images. + seq_length (int): The maximum sequence length for tokenized text. + micro_batch_size (int): The batch size for training and validation. + num_workers (int): Number of workers for data loading. + pin_memory (bool): Whether to pin memory in the DataLoader. + multimodal_sample_config (MultiModalSampleConfig): Configuration object for multimodal samples. + task_encoder (MultiModalTaskEncoder): Encoder responsible for encoding and batching samples. + init_global_step (int): The initial global step for the trainer, used for resuming training. + data_sampler (SequentialMegatronSampler): Sampler responsible for generating sequential samples. + train_dataloader_object (Optional): The DataLoader object for training data. + val_dataloader_object (Optional): The DataLoader object for validation data. + """ + + def __init__( + self, + path: str, + seq_length: int = 2048, + micro_batch_size: int = 1, + global_batch_size: int = 8, + num_workers: int = 1, + pin_memory: bool = True, + task_encoder: DefaultTaskEncoder = None, + use_train_split_for_val: bool = False, + ) -> None: + """ + Initialize the SimpleMultiModalDataModule. + + Parameters: + path (str): Path to the dataset. + tokenizer (Tokenizer): The tokenizer used for processing text. + image_processor (ImageProcessor): The image processor used for preprocessing images. + seq_length (int, optional): The maximum sequence length for tokenized text. Defaults to 2048. + micro_batch_size (int, optional): The batch size for training and validation. Defaults to 1. + num_workers (int, optional): Number of workers for data loading. Defaults to 1. + pin_memory (bool, optional): Whether to pin memory in the DataLoader. Defaults to True. + """ + + super().__init__( + path=path, + tokenizer=None, + image_processor=None, + seq_length=seq_length, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + num_workers=num_workers, + pin_memory=pin_memory, + task_encoder=task_encoder, + ) + self.use_train_split_for_val = use_train_split_for_val + + def datasets_provider(self, worker_config, split: Literal["train", "val"] = "val"): + """ + Provide the dataset for training or validation. + + This method retrieves the dataset for the specified split (either 'train' or 'val') and configures + it according to the worker configuration. + + Parameters: + worker_config: Configuration for the data loader workers. + split (Literal['train', 'val'], optional): The data split to retrieve ('train' or 'val'). Defaults to 'val'. + + Returns: + Dataset: The dataset configured for the specified split. + """ + if split not in {"train", "val"}: + raise ValueError("Invalid value for split. Allowed values are 'train' or 'val'.") + if self.use_train_split_for_val: + split = "train" + _dataset = get_train_dataset( + self.path, + batch_size=self.micro_batch_size, + task_encoder=self.task_encoder, + worker_config=worker_config, + max_samples_per_sequence=None, + shuffle_buffer_size=100, + split_part=split, + batch_drop_last=True, + virtual_epoch_length=1_000_000_000, # a hack to avoid energon end of epoch warning + ) + return _dataset + + def val_dataloader(self) -> EVAL_DATALOADERS: + """ + Configure the validation DataLoader. + + This method configures the DataLoader for validation data. + + Parameters: + worker_config: Configuration for the data loader workers. + + Returns: + DataLoader: The DataLoader for validation data. + """ + if self.use_train_split_for_val: + return self.train_dataloader() + return super().val_dataloader() + + def load_state_dict(self, state_dict: Dict[str, Any]) -> None: + """ + Load the state of the data module from a checkpoint. + + This method is called when loading a checkpoint. It restores the state of the data module, + including the state of the dataloader and the number of consumed samples. + + Parameters: + state_dict (Dict[str, Any]): The state dictionary containing the saved state of the data module. + """ + try: + super().load_state_dict(state_dict) + except Exception as e: + logging.warning(f"datamodule.load_state_dict failed {e}") diff --git a/nemo_vfm/diffusion/data/diffusion_fake_datamodule.py b/nemo_vfm/diffusion/data/diffusion_fake_datamodule.py new file mode 100644 index 00000000..f40d29ca --- /dev/null +++ b/nemo_vfm/diffusion/data/diffusion_fake_datamodule.py @@ -0,0 +1,215 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import lightning.pytorch as pl +import torch +from lightning.pytorch.utilities.types import EVAL_DATALOADERS, TRAIN_DATALOADERS +from nemo.collections.diffusion.models.model import DiTConfig +from nemo.lightning.pytorch.plugins import MegatronDataSampler +from torch.utils.data import DataLoader + + +class PosEmb3D: + """Generates and provides 3D positional embeddings for video data.""" + + def __init__(self, *, max_t=96, max_h=960, max_w=960): + self.max_t = max_t + self.max_h = max_h + self.max_w = max_w + self.generate_pos_id() + + def generate_pos_id(self): + """Generates the positional ID grid based on max_t, max_h, and max_w.""" + self.grid = torch.stack( + torch.meshgrid( + torch.arange(self.max_t, device="cpu"), + torch.arange(self.max_h, device="cpu"), + torch.arange(self.max_w, device="cpu"), + ), + dim=-1, + ) + + def get_pos_id_3d(self, *, t, h, w): + """Retrieves a subset of the positional IDs for the specified dimensions. + + Parameters: + t (int): Number of time frames. + h (int): Height dimension. + w (int): Width dimension. + + Returns: + torch.Tensor: The positional IDs tensor with shape (t, h, w, 3). + """ + if t > self.max_t or h > self.max_h or w > self.max_w: + self.max_t = max(self.max_t, t) + self.max_h = max(self.max_h, h) + self.max_w = max(self.max_w, w) + self.generate_pos_id() + return self.grid[:t, :h, :w] + + +class DiTVideoLatentFakeDataset(torch.utils.data.Dataset): + """A fake dataset for generating synthetic video latent data.""" + + def __init__( + self, + n_frames, + max_h, + max_w, + patch_size, + in_channels, + crossattn_emb_size, + max_text_seqlen=512, + seq_length=8192, + ): + self.max_t = n_frames + self.max_height = max_h + self.max_width = max_w + self.patch_size = patch_size + self.in_channels = in_channels + self.text_dim = crossattn_emb_size + self.text_seqlen = max_text_seqlen + self.seq_length = seq_length + + def __len__(self): + """Returns the total number of samples.""" + return 100000000 + + def __getitem__(self, idx): + """Generates a single sample of data. + + Parameters: + idx (int): Index of the data sample. + + Returns: + dict: A dictionary containing video latent data and related information. + """ + # t = self.max_t + # h = self.max_height + # w = self.max_width + p = self.patch_size + c = self.in_channels + + video_latent = torch.ones(self.seq_length, c * p**2, dtype=torch.bfloat16) * 0.5 + text_embedding = torch.randn(self.text_seqlen, self.text_dim, dtype=torch.bfloat16) + # pos_emb = pos_id_3d.get_pos_id_3d(t=t, h=h // p, w=w // p).reshape(-1, 3) + + return { + "video": video_latent, + "t5_text_embeddings": text_embedding, + "seq_len_q": torch.tensor([video_latent.shape[0]], dtype=torch.int32).squeeze(), + "seq_len_kv": torch.tensor([self.text_seqlen], dtype=torch.int32).squeeze(), + "pos_ids": torch.zeros((self.seq_length, 3), dtype=torch.int32), + "loss_mask": torch.ones(video_latent.shape[0], dtype=torch.bfloat16), + } + + def _collate_fn(self, batch): + """A default implementation of a collation function. + + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user passes as a functor to DataLoader. + + The method optionally performs neural type checking and adds types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns: + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +class VideoLatentFakeDataModule(pl.LightningDataModule): + """A LightningDataModule for generating fake video latent data for training.""" + + def __init__( + self, + model_config: DiTConfig, + seq_length: int = 2048, + micro_batch_size: int = 1, + global_batch_size: int = 8, + num_workers: int = 1, + pin_memory: bool = True, + task_encoder=None, + use_train_split_for_val: bool = False, + ) -> None: + super().__init__() + self.seq_length = seq_length + self.micro_batch_size = micro_batch_size + self.global_batch_size = global_batch_size + self.num_workers = num_workers + self.model_config = model_config + + self.data_sampler = MegatronDataSampler( + seq_len=self.seq_length, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + ) + + def setup(self, stage: str = "") -> None: + """Sets up the dataset for training and validation. + + Parameters: + stage (str): Optional stage argument (unused). + """ + self._train_ds = DiTVideoLatentFakeDataset( + n_frames=self.model_config.max_frames, + max_h=self.model_config.max_img_h, + max_w=self.model_config.max_img_w, + patch_size=self.model_config.patch_spatial, + in_channels=self.model_config.in_channels, + crossattn_emb_size=self.model_config.crossattn_emb_size, + ) + + def train_dataloader(self) -> TRAIN_DATALOADERS: + """Returns the training DataLoader.""" + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def val_dataloader(self) -> EVAL_DATALOADERS: + """Returns the validation DataLoader.""" + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def _create_dataloader(self, dataset, **kwargs) -> DataLoader: + """Creates a DataLoader for the given dataset. + + Parameters: + dataset (Dataset): The dataset to load. + **kwargs: Additional arguments for DataLoader. + + Returns: + DataLoader: The DataLoader instance. + """ + return DataLoader( + dataset, + num_workers=self.num_workers, + pin_memory=True, + persistent_workers=True, + collate_fn=dataset.collate_fn, + **kwargs, + ) diff --git a/nemo_vfm/diffusion/data/diffusion_mock_datamodule.py b/nemo_vfm/diffusion/data/diffusion_mock_datamodule.py new file mode 100644 index 00000000..73c4208a --- /dev/null +++ b/nemo_vfm/diffusion/data/diffusion_mock_datamodule.py @@ -0,0 +1,277 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import List, Optional + +import lightning.pytorch as pl +import torch +from lightning.pytorch.utilities.types import EVAL_DATALOADERS, TRAIN_DATALOADERS +from nemo.lightning.pytorch.plugins import MegatronDataSampler +from torch.utils.data import DataLoader, Dataset + + +class MockDataModule(pl.LightningDataModule): + """ + A PyTorch Lightning DataModule for creating mock datasets for training, validation, and testing. + + Args: + image_h (int): Height of the images in the dataset. Default is 1024. + image_w (int): Width of the images in the dataset. Default is 1024. + micro_batch_size (int): Micro batch size for the data sampler. Default is 4. + global_batch_size (int): Global batch size for the data sampler. Default is 8. + rampup_batch_size (Optional[List[int]]): Ramp-up batch size for the data sampler. Default is None. + num_train_samples (int): Number of training samples. Default is 10,000. + num_val_samples (int): Number of validation samples. Default is 10,000. + num_test_samples (int): Number of testing samples. Default is 10,000. + num_workers (int): Number of worker threads for data loading. Default is 8. + pin_memory (bool): Whether to use pinned memory for data loading. Default is True. + persistent_workers (bool): Whether to use persistent workers for data loading. Default is False. + image_precached (bool): Whether the images are pre-cached. Default is False. + text_precached (bool): Whether the text data is pre-cached. Default is False. + """ + + def __init__( + self, + image_h: int = 1024, + image_w: int = 1024, + micro_batch_size: int = 4, + global_batch_size: int = 8, + rampup_batch_size: Optional[List[int]] = None, + num_train_samples: int = 10_000, + num_val_samples: int = 10_000, + num_test_samples: int = 10_000, + num_workers: int = 8, + pin_memory: bool = True, + persistent_workers: bool = False, + image_precached=False, + text_precached=False, + ): + super().__init__() + self.image_h = image_h + self.image_w = image_w + self.num_train_samples = num_train_samples + self.num_val_samples = num_val_samples + self.num_test_samples = num_test_samples + self.num_workers = num_workers + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers + self.image_precached = image_precached + self.text_precached = text_precached + self.global_batch_size = global_batch_size + + self.data_sampler = MegatronDataSampler( + seq_len=10, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + rampup_batch_size=rampup_batch_size, + ) + + def setup(self, stage: str = "") -> None: + """ + Sets up datasets for training, validation, and testing. + + Args: + stage (str): The stage of the process (e.g., 'fit', 'test'). Default is an empty string. + """ + self._train_ds = _MockT2IDataset( + image_H=1024, + image_W=1024, + length=self.num_train_samples, + image_precached=self.image_precached, + text_precached=self.text_precached, + ) + self._validation_ds = _MockT2IDataset( + image_H=1024, + image_W=1024, + length=self.num_val_samples, + image_precached=self.image_precached, + text_precached=self.text_precached, + ) + self._test_ds = _MockT2IDataset( + image_H=1024, + image_W=1024, + length=self.num_test_samples, + image_precached=self.image_precached, + text_precached=self.text_precached, + ) + + def train_dataloader(self) -> TRAIN_DATALOADERS: + """ + Returns the training DataLoader. + + Returns: + TRAIN_DATALOADERS: DataLoader for the training dataset. + """ + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def val_dataloader(self) -> EVAL_DATALOADERS: + """ + Returns the validation DataLoader. + + Returns: + EVAL_DATALOADERS: DataLoader for the validation dataset. + """ + if not hasattr(self, "_validation_ds"): + self.setup() + return self._create_dataloader(self._validation_ds) + + def test_dataloader(self) -> EVAL_DATALOADERS: + """ + Returns the testing DataLoader. + + Returns: + EVAL_DATALOADERS: DataLoader for the testing dataset. + """ + if not hasattr(self, "_test_ds"): + self.setup() + return self._create_dataloader(self._test_ds) + + def _create_dataloader(self, dataset, **kwargs) -> DataLoader: + """ + Creates a DataLoader for the given dataset. + + Args: + dataset: The dataset to load. + **kwargs: Additional arguments for the DataLoader. + + Returns: + DataLoader: Configured DataLoader for the dataset. + """ + return DataLoader( + dataset, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers, + **kwargs, + ) + + +class _MockT2IDataset(Dataset): + """ + A mock dataset class for text-to-image tasks, simulating data samples for training and testing. + + This dataset generates synthetic data for both image and text inputs, with options to use + pre-cached latent representations or raw data. The class is designed for use in testing and + prototyping machine learning models. + + Attributes: + image_H (int): Height of the generated images. + image_W (int): Width of the generated images. + length (int): Total number of samples in the dataset. + image_key (str): Key for accessing image data in the output dictionary. + txt_key (str): Key for accessing text data in the output dictionary. + hint_key (str): Key for accessing hint data in the output dictionary. + image_precached (bool): Whether to use pre-cached latent representations for images. + text_precached (bool): Whether to use pre-cached embeddings for text. + prompt_seq_len (int): Sequence length for text prompts. + pooled_prompt_dim (int): Dimensionality of pooled text embeddings. + context_dim (int): Dimensionality of the text embedding context. + vae_scale_factor (int): Scaling factor for the VAE latent representation. + vae_channels (int): Number of channels in the VAE latent representation. + latent_shape (tuple): Shape of the latent representation for images (if pre-cached). + prompt_embeds_shape (tuple): Shape of the text prompt embeddings (if pre-cached). + pooped_prompt_embeds_shape (tuple): Shape of pooled text embeddings (if pre-cached). + text_ids_shape (tuple): Shape of the text token IDs (if pre-cached). + + Methods: + __getitem__(index): + Retrieves a single sample from the dataset based on the specified index. + __len__(): + Returns the total number of samples in the dataset. + """ + + def __init__( + self, + image_H, + image_W, + length=100000, + image_key="images", + txt_key="txt", + hint_key="hint", + image_precached=False, + text_precached=False, + prompt_seq_len=256, + pooled_prompt_dim=768, + context_dim=4096, + vae_scale_factor=8, + vae_channels=16, + ): + super().__init__() + self.length = length + self.H = image_H + self.W = image_W + self.image_key = image_key + self.txt_key = txt_key + self.hint_key = hint_key + self.image_precached = image_precached + self.text_precached = text_precached + if self.image_precached: + self.latent_shape = (vae_channels, int(image_H // vae_scale_factor), int(image_W // vae_scale_factor)) + if self.text_precached: + self.prompt_embeds_shape = (prompt_seq_len, context_dim) + self.pooped_prompt_embeds_shape = (pooled_prompt_dim,) + self.text_ids_shape = (prompt_seq_len, 3) + + def __getitem__(self, index): + """ + Retrieves a single sample from the dataset. + + The sample can include raw image and text data or pre-cached latent representations, + depending on the configuration. + + Args: + index (int): Index of the sample to retrieve. + + Returns: + dict: A dictionary containing the generated data sample. The keys and values + depend on whether `image_precached` and `text_precached` are set. + Possible keys include: + - 'latents': Pre-cached latent representation of the image. + - 'control_latents': Pre-cached control latent representation. + - 'images': Raw image tensor. + - 'hint': Hint tensor for the image. + - 'prompt_embeds': Pre-cached text prompt embeddings. + - 'pooled_prompt_embeds': Pooled text prompt embeddings. + - 'text_ids': Text token IDs. + - 'txt': Text input string (if text is not pre-cached). + """ + item = {} + if self.image_precached: + item["latents"] = torch.randn(self.latent_shape) + item["control_latents"] = torch.randn(self.latent_shape) + else: + item[self.image_key] = torch.randn(3, self.H, self.W) + item[self.hint_key] = torch.randn(3, self.H, self.W) + + if self.text_precached: + item["prompt_embeds"] = torch.randn(self.prompt_embeds_shape) + item["pooled_prompt_embeds"] = torch.randn(self.pooped_prompt_embeds_shape) + item["text_ids"] = torch.randn(self.text_ids_shape) + else: + item[self.txt_key] = "This is a sample caption input" + + return item + + def __len__(self): + """ + Returns the total number of samples in the dataset. + + Returns: + int: Total number of samples (`length` attribute). + """ + return self.length diff --git a/nemo_vfm/diffusion/data/diffusion_taskencoder.py b/nemo_vfm/diffusion/data/diffusion_taskencoder.py new file mode 100644 index 00000000..ad046f26 --- /dev/null +++ b/nemo_vfm/diffusion/data/diffusion_taskencoder.py @@ -0,0 +1,245 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +import torch.nn.functional as F +from einops import rearrange +from megatron.energon import DefaultTaskEncoder, SkipSample +from megatron.energon.task_encoder.cooking import Cooker, basic_sample_keys +from nemo.lightning.io.mixin import IOMixin + + +def cook(sample: dict) -> dict: + """ + Processes a raw sample dictionary from energon dataset and returns a new dictionary with specific keys. + + Args: + sample (dict): The input dictionary containing the raw sample data. + + Returns: + dict: A new dictionary containing the processed sample data with the following keys: + - All keys from the result of `basic_sample_keys(sample)` + - 'json': The contains meta data like resolution, aspect ratio, fps, etc. + - 'pth': contains video latent tensor + - 'pickle': contains text embeddings + """ + return dict( + **basic_sample_keys(sample), + json=sample[".json"], + pth=sample[".pth"], + pickle=sample[".pickle"], + ) + + +class BasicDiffusionTaskEncoder(DefaultTaskEncoder, IOMixin): + """ + BasicDiffusionTaskEncoder is a class that encodes image/video samples for diffusion tasks. + Attributes: + cookers (list): A list of Cooker objects used for processing. + max_frames (int, optional): The maximum number of frames to consider from the video. Defaults to None. + text_embedding_padding_size (int): The padding size for text embeddings. Defaults to 512. + Methods: + __init__(*args, max_frames=None, text_embedding_padding_size=512, **kwargs): + Initializes the BasicDiffusionTaskEncoder with optional maximum frames and text embedding padding size. + encode_sample(sample: dict) -> dict: + Encodes a given sample dictionary containing video and text data. + Args: + sample (dict): A dictionary containing 'pth' for video latent and 'json' for additional info. + Returns: + dict: A dictionary containing encoded video, text embeddings, text mask, and loss mask. + Raises: + SkipSample: If the video latent contains NaNs, Infs, or is not divisible by the tensor parallel size. + """ + + cookers = [ + Cooker(cook), + ] + + def __init__( + self, + *args, + max_frames: int = None, + text_embedding_padding_size: int = 512, + seq_length: int = None, + patch_spatial: int = 2, + patch_temporal: int = 1, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.max_frames = max_frames + self.text_embedding_padding_size = text_embedding_padding_size + self.seq_length = seq_length + self.patch_spatial = patch_spatial + self.patch_temporal = patch_temporal + + def encode_sample(self, sample: dict) -> dict: + video_latent = sample["pth"] + + if torch.isnan(video_latent).any() or torch.isinf(video_latent).any(): + raise SkipSample() + if torch.max(torch.abs(video_latent)) > 1e3: + raise SkipSample() + + info = sample["json"] + C, T, H, W = video_latent.shape + seq_len = ( + video_latent.shape[-1] + * video_latent.shape[-2] + * video_latent.shape[-3] + // self.patch_spatial**2 + // self.patch_temporal + ) + is_image = T == 1 + + if seq_len > self.seq_length: + raise SkipSample() + + if self.max_frames is not None: + video_latent = video_latent[:, : self.max_frames, :, :] + + # tpcp_size = parallel_state.get_tensor_model_parallel_world_size() + # if parallel_state.get_context_parallel_world_size() > 1: + # tpcp_size *= parallel_state.get_context_parallel_world_size() * 2 + # if (T * H * W) % tpcp_size != 0: + # warnings.warn(f'skipping {video_latent.shape=} not divisible by {tpcp_size=}') + # raise SkipSample() + + video_latent = rearrange( + video_latent, + "C (T pt) (H ph) (W pw) -> (T H W) (ph pw pt C)", + ph=self.patch_spatial, + pw=self.patch_spatial, + pt=self.patch_temporal, + ) + + if is_image: + t5_text_embeddings = torch.from_numpy(sample["pickle"]).to(torch.bfloat16) + else: + t5_text_embeddings = torch.from_numpy(sample["pickle"][0]).to(torch.bfloat16) + t5_text_embeddings_seq_length = t5_text_embeddings.shape[0] + + if t5_text_embeddings_seq_length > self.text_embedding_padding_size: + t5_text_embeddings = t5_text_embeddings[: self.text_embedding_padding_size] + else: + t5_text_embeddings = F.pad( + t5_text_embeddings, + ( + 0, + 0, + 0, + self.text_embedding_padding_size - t5_text_embeddings_seq_length, + ), + ) + t5_text_mask = torch.ones(t5_text_embeddings_seq_length, dtype=torch.bfloat16) + + if is_image: + h, w = info["image_height"], info["image_width"] + fps = torch.tensor([30] * 1, dtype=torch.bfloat16) + num_frames = torch.tensor([1] * 1, dtype=torch.bfloat16) + else: + h, w = info["height"], info["width"] + fps = torch.tensor([info["framerate"]] * 1, dtype=torch.bfloat16) + num_frames = torch.tensor([info["num_frames"]] * 1, dtype=torch.bfloat16) + image_size = torch.tensor([[h, w, h, w]] * 1, dtype=torch.bfloat16) + + pos_ids = rearrange( + pos_id_3d.get_pos_id_3d(t=T // self.patch_temporal, h=H // self.patch_spatial, w=W // self.patch_spatial), + "T H W d -> (T H W) d", + ) + + if self.seq_length is not None: + pos_ids = F.pad(pos_ids, (0, 0, 0, self.seq_length - seq_len)) + loss_mask = torch.zeros(self.seq_length, dtype=torch.bfloat16) + loss_mask[:seq_len] = 1 + video_latent = F.pad(video_latent, (0, 0, 0, self.seq_length - seq_len)) + else: + loss_mask = torch.ones(seq_len, dtype=torch.bfloat16) + + return dict( + video=video_latent, + t5_text_embeddings=t5_text_embeddings, + t5_text_mask=t5_text_mask, + image_size=image_size, + fps=fps, + num_frames=num_frames, + loss_mask=loss_mask, + seq_len_q=torch.tensor(seq_len, dtype=torch.int32), + seq_len_kv=torch.tensor(self.text_embedding_padding_size, dtype=torch.int32), + pos_ids=pos_ids, + latent_shape=torch.tensor([C, T, H, W], dtype=torch.int32), + ) + + +class PosID3D: + def __init__(self, *, max_t=32, max_h=128, max_w=128): + self.max_t = max_t + self.max_h = max_h + self.max_w = max_w + self.generate_pos_id() + + def generate_pos_id(self): + self.grid = torch.stack( + torch.meshgrid( + torch.arange(self.max_t, device="cpu"), + torch.arange(self.max_h, device="cpu"), + torch.arange(self.max_w, device="cpu"), + ), + dim=-1, + ) + + def get_pos_id_3d(self, *, t, h, w): + if t > self.max_t or h > self.max_h or w > self.max_w: + self.max_t = max(self.max_t, t) + self.max_h = max(self.max_h, h) + self.max_w = max(self.max_w, w) + self.generate_pos_id() + return self.grid[:t, :h, :w] + + +pos_id_3d = PosID3D() + + +def cook_raw_iamges(sample: dict) -> dict: + """ + Processes a raw sample dictionary from energon dataset and returns a new dictionary with specific keys. + + Args: + sample (dict): The input dictionary containing the raw sample data. + + Returns: + dict: A new dictionary containing the processed sample data with the following keys: + - All keys from the result of `basic_sample_keys(sample)` + - 'jpg': original images + - 'png': contains control images + - 'txt': contains raw text + """ + return dict( + **basic_sample_keys(sample), + images=sample["jpg"], + hint=sample["png"], + txt=sample["txt"], + ) + + +class RawImageDiffusionTaskEncoder(DefaultTaskEncoder, IOMixin): + """ + Dummy task encoder takes raw image input on CrudeDataset. + """ + + cookers = [ + # Cooker(cook), + Cooker(cook_raw_iamges), + ] diff --git a/nemo_vfm/diffusion/data/prepare_energon_dataset.py b/nemo_vfm/diffusion/data/prepare_energon_dataset.py new file mode 100644 index 00000000..2ea4b12d --- /dev/null +++ b/nemo_vfm/diffusion/data/prepare_energon_dataset.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +import pickle +from typing import Callable, List + +import nemo_run as run +import numpy as np +import torch +import torch.distributed as dist +import webdataset as wds + + +def get_start_end_idx_for_this_rank(dataset_size, rank, world_size): + """ + Calculate the start and end indices for a given rank in a distributed setting. + + Args: + dataset_size (int): The total size of the dataset. + rank (int): The rank of the current process. + world_size (int): The total number of processes. + + Returns: + tuple: A tuple containing the start index (int) and end index (int) for the given rank. + """ + split_size = dataset_size // world_size + start_idx = rank * split_size + # The last rank takes the remainder + end_idx = start_idx + split_size if rank != world_size - 1 else dataset_size + return start_idx, end_idx + + +def dummy_process_func(input): + """ + Generates a sample dictionary containing random image latent tensor, text embedding, + and metadata based on the provided input key. + + Args: + input (str): The key to be used in the sample dictionary. + + Returns: + dict: A dictionary containing the following keys: + - "__key__": The input key. + - ".pth": A randomly generated image latent tensor with shape (3, 1, 720, 1280) and dtype torch.bfloat16. + - ".pickle": A pickled numpy array representing a random text embedding with shape (512, 2048). + - ".json": A dictionary containing metadata with keys: + - "image_height": The height of the image (720). + - "image_width": The width of the image (1280). + """ + C, T, H, W = 3, 1, 720, 1280 + image_latent = torch.randn(C, T, H, W, dtype=torch.bfloat16) + text_embedding = np.random.randn(512, 2048) + sample = { + "__key__": input, + ".pth": image_latent, + ".pickle": pickle.dumps(text_embedding), + ".json": { + "image_height": H, + "image_width": W, + }, + } + return sample + + +@torch.no_grad() +@run.cli.entrypoint +def prepare(process_func: Callable, inputs: List[str], output_dir: str = "output"): + """ + distributed prepration webdataset using the provided processing function, and writes the processed samples to tar files. + + Args: + process_func (Callable): A function that processes a single input and returns the processed sample. + inputs (List[str]): A list of input file paths or data entries to be processed. + output_dir (str, optional): The directory where the output tar files will be saved. Defaults to 'output'. + """ + rank = dist.get_rank() + world_size = torch.distributed.get_world_size() + + start_idx, end_idx = get_start_end_idx_for_this_rank(len(inputs), rank, world_size) + os.makedirs(output_dir, exist_ok=True) + output_tar = os.path.join(output_dir, f"rank{rank}-%06d.tar") + with wds.ShardWriter(output_tar, maxcount=10000) as sink: + for i in range(start_idx, end_idx): + sample = process_func(inputs[i]) + # Write the sample to the tar file + sink.write(sample) + + +@run.cli.factory(target=prepare) +def prepare_dummy_image_dataset() -> run.Partial: + recipe = run.Partial( + prepare, + process_func=dummy_process_func, + inputs=list(str(i) + ".jpg" for i in range(10000)), + ) + return recipe + + +if __name__ == "__main__": + dist.init_process_group("nccl") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + run.cli.main(prepare, default_factory=prepare_dummy_image_dataset) diff --git a/nemo_vfm/diffusion/data/readme.rst b/nemo_vfm/diffusion/data/readme.rst new file mode 100644 index 00000000..57a17379 --- /dev/null +++ b/nemo_vfm/diffusion/data/readme.rst @@ -0,0 +1,26 @@ +Preparing Image / Video Megatron Energon WebDataset with Cosmos Tokenizer +=========================== + +This script is an example on preparing a WebDataset for an image / video + text dataset using distributed processing with the Cosmos Tokenizer. It processes each sample by generating a **continuous** image / video latent using the Cosmos video tokenizer and a T5 embedding from the text caption. Then, the processed data is stored in a WebDataset-compatible format. + +Requirements +------------ +- **Dependencies**: + - Please use the latest NeMo dev container: ``nvcr.io/nvidia/nemo:dev`` + - You may also need to install ``jammy`` and ``mediapy`` depending on your dev container version. + +- **Data**: + - The script uses an example dataset that comes in parquet format. To use a custom, you will need to write a custom ``process_func`` and create a new factory recipe that uses your new ``process_func``. + +Usage +----- +1. **Set up your environment**: + Pull and launch the NeMo dev container to run your script. + +2. **Customize Cache Path**: + Set the T5 cache directory path in the script by specifying the `t5_cache_dir` variable. + +3. **Running the Script**: + To run the script on 8 GPUs, use the following command: + + ``bash torchrun --nproc_per_node=8 nemo/collections/diffusion/data/prepare_energon_dataset.py`` diff --git a/nemo_vfm/diffusion/data/test_datamodule.py b/nemo_vfm/diffusion/data/test_datamodule.py new file mode 100644 index 00000000..0034f752 --- /dev/null +++ b/nemo_vfm/diffusion/data/test_datamodule.py @@ -0,0 +1,92 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +import time + +import fiddle as fdl +import numpy as np +import pytest +import torch +from megatron.core import parallel_state +from nemo.collections.diffusion.data.diffusion_taskencoder import BasicDiffusionTaskEncoder +from nemo.collections.diffusion.train import multimodal_datamodule +from tqdm import tqdm + + +# Fixture to initialize distributed training only once +@pytest.fixture(scope="session", autouse=True) +def initialize_distributed(): + if not torch.distributed.is_initialized(): + rank = int(os.environ["LOCAL_RANK"]) + world_size = torch.cuda.device_count() + torch.cuda.set_device(rank) + torch.distributed.init_process_group(backend="nccl", world_size=world_size, rank=rank) + parallel_state.initialize_model_parallel() + + +# Fixture to get the value of the custom command-line option +@pytest.fixture +def path(): + return os.getenv("DATA_DIR") + + +def test_datamodule(path): + config = multimodal_datamodule() + config.path = path + config.num_workers = 120 + config.seq_length = 260 + config.task_encoder.seq_length = 260 + datamodule = fdl.build(config) + # datamodule = SimpleMultiModalDataModule( + # path=path, + # seq_length=260, + # micro_batch_size=1, + # num_workers=256, + # tokenizer=None, + # image_processor=None, + # task_encoder=BasicDiffusionTaskEncoder(seq_length=260, text_embedding_padding_size=512, + # ), + # ) + + for i, batch in enumerate(datamodule.train_dataloader()): + print(batch["seq_len_q"]) + if i == 1: + start_time = time.time() + if i > 100: + break + + elapsed_time = time.time() - start_time + print(f"Elapsed time for loading 100 batches: {elapsed_time} seconds, {elapsed_time / 100} seconds per batch") + + +def test_taskencoder(): + taskencoder = BasicDiffusionTaskEncoder( + text_embedding_padding_size=512, + seq_length=260, + ) + + start_time = time.time() + for _ in tqdm(range(100)): + sample = { + "pth": torch.randn(3, 1, 30, 30), + "pickle": np.random.randn(256, 1024), + "json": {"image_height": 1, "image_width": 1}, + } + taskencoder.encode_sample(sample) + + elapsed_time = time.time() - start_time + print(f"Elapsed time for loading 100 batches: {elapsed_time} seconds") diff --git a/nemo_vfm/diffusion/data/utils.py b/nemo_vfm/diffusion/data/utils.py new file mode 100644 index 00000000..dbe8ebad --- /dev/null +++ b/nemo_vfm/diffusion/data/utils.py @@ -0,0 +1,203 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import numpy as np + + +def minimal_crop(tensor, target_divisor): + """ + Crops the input tensor minimally so that the total number of elements + (T * H * W) is divisible by the specified target_divisor. + + Parameters: + - tensor: NumPy array of shape (C, T, H, W) + - target_divisor: Positive integer specifying the desired divisor + + Returns: + - cropped_tensor: Cropped tensor meeting the divisibility requirement + + Raises: + - ValueError: If it's impossible to meet the divisibility requirement + """ + if not isinstance(target_divisor, int) or target_divisor <= 0: + raise ValueError("target_divisor must be a positive integer greater than zero.") + + C, T, H, W = tensor.shape + total_elements = T * H * W + remainder = total_elements % target_divisor + + if remainder == 0: + return tensor # No cropping needed + + # Elements per unit length in each dimension + elements_per_T = H * W + elements_per_H = T * W + elements_per_W = T * H + + min_elements_removed = None + optimal_deltas = None + + # Limit the search range to avoid unnecessary computations + max_delta_T = min(T - 1, (remainder // elements_per_T) + 1) + max_delta_H = min(H - 1, (remainder // elements_per_H) + 1) + max_delta_W = min(W - 1, (remainder // elements_per_W) + 1) + + for delta_T in range(0, max_delta_T + 1): + for delta_H in range(0, max_delta_H + 1): + for delta_W in range(0, max_delta_W + 1): + if delta_T == delta_H == delta_W == 0: + continue # No cropping + + new_T = T - delta_T + new_H = H - delta_H + new_W = W - delta_W + + if new_T <= 0 or new_H <= 0 or new_W <= 0: + continue # Invalid dimensions + + new_total_elements = new_T * new_H * new_W + if new_total_elements % target_divisor == 0: + elements_removed = delta_T * elements_per_T + delta_H * elements_per_H + delta_W * elements_per_W + if min_elements_removed is None or elements_removed < min_elements_removed: + min_elements_removed = elements_removed + optimal_deltas = (delta_T, delta_H, delta_W) + + if optimal_deltas is None: + raise ValueError("Cannot crop tensor to meet divisibility requirement.") + + delta_T, delta_H, delta_W = optimal_deltas + + # Perform the cropping + # T dimension: crop from the end + end_T = T - delta_T + + # H dimension: center crop + start_H = delta_H // 2 + end_H = H - (delta_H - delta_H // 2) + + # W dimension: center crop + start_W = delta_W // 2 + end_W = W - (delta_W - delta_W // 2) + + cropped_tensor = tensor[:, :end_T, start_H:end_H, start_W:end_W] + return cropped_tensor + + +def test_no_cropping_needed(): + """Test when the tensor already meets the divisibility requirement.""" + C, T, H, W = 3, 8, 8, 8 + target_divisor = 8 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + assert cropped_tensor.shape == (C, T, H, W) + assert (T * H * W) % target_divisor == 0 + + +def test_minimal_cropping_T_dimension(): + """Test minimal cropping along the T dimension.""" + C, T, H, W = 3, 9, 7, 6 + target_divisor = 8 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + new_T = cropped_tensor.shape[1] + assert new_T == T - 1, cropped_tensor.shape + assert (new_T * H * W) % target_divisor == 0 + + +def test_minimal_cropping_H_dimension(): + """Test minimal cropping along the H dimension.""" + C, T, H, W = 3, 7, 9, 6 + target_divisor = 8 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + new_H = cropped_tensor.shape[2] + assert new_H == H - 1, cropped_tensor.shape + assert (T * new_H * W) % target_divisor == 0 + + +def test_minimal_cropping_W_dimension(): + """Test minimal cropping along the W dimension.""" + C, T, H, W = 3, 4, 3, 9 + target_divisor = 8 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + new_W = cropped_tensor.shape[3] + assert new_W == W - 1, cropped_tensor.shape + assert (T * H * new_W) % target_divisor == 0 + + +def test_cropping_multiple_dimensions(): + """Test when minimal cropping requires adjustments on multiple dimensions.""" + C, T, H, W = 3, 9, 9, 8 + target_divisor = 16 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + new_T, new_H, new_W = cropped_tensor.shape[1:] + assert new_T <= T and new_H <= H and new_W <= W + assert (new_T * new_H * new_W) % target_divisor == 0 + + +def test_large_tensor_high_divisor(): + """Test with a larger tensor and higher target_divisor.""" + C, T, H, W = 3, 50, 50, 50 + target_divisor = 1024 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + total_elements = cropped_tensor.shape[1] * cropped_tensor.shape[2] * cropped_tensor.shape[3] + assert total_elements % target_divisor == 0 + + +def test_impossible_cropping(): + """Test that an error is raised when it's impossible to meet the requirement.""" + C, T, H, W = 3, 1, 1, 1 + target_divisor = 2 + tensor = np.zeros((C, T, H, W)) + try: + minimal_crop(tensor, target_divisor) + except ValueError: + pass + + +def test_invalid_target_divisor(): + """Test that an error is raised when target_divisor is invalid.""" + C, T, H, W = 3, 8, 8, 8 + tensor = np.zeros((C, T, H, W)) + try: + minimal_crop(tensor, -1) + except ValueError: + pass + + +def test_minimal_elements_removed(): + """Test that the minimal number of elements are removed.""" + C, T, H, W = 3, 7, 7, 7 + target_divisor = 8 + tensor = np.zeros((C, T, H, W)) + cropped_tensor = minimal_crop(tensor, target_divisor) + elements_removed = (T * H * W) - (cropped_tensor.shape[1] * cropped_tensor.shape[2] * cropped_tensor.shape[3]) + print(cropped_tensor.shape) + assert elements_removed > 0 + assert (cropped_tensor.shape[1] * cropped_tensor.shape[2] * cropped_tensor.shape[3]) % target_divisor == 0 + + +test_no_cropping_needed() +test_minimal_elements_removed() +test_cropping_multiple_dimensions() +test_minimal_cropping_T_dimension() +test_minimal_cropping_H_dimension() +test_minimal_cropping_W_dimension() +test_impossible_cropping() +test_invalid_target_divisor() diff --git a/nemo_vfm/diffusion/datamodule.py b/nemo_vfm/diffusion/datamodule.py new file mode 100644 index 00000000..56b30da0 --- /dev/null +++ b/nemo_vfm/diffusion/datamodule.py @@ -0,0 +1,529 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import json +import os +import random +from typing import Dict, Literal + +import torch +from nemo.collections.llm.gpt.data.mock import MockDataModule +from torch.utils.data import Dataset + + +class PosEmb3D: + def __init__(self, *, max_t=96, max_h=960, max_w=960): + self.max_t = max_t + self.max_h = max_h + self.max_w = max_w + self.generate_pos_id() + + def generate_pos_id(self): + self.grid = torch.stack( + torch.meshgrid( + torch.arange(self.max_t, device="cpu"), + torch.arange(self.max_h, device="cpu"), + torch.arange(self.max_w, device="cpu"), + ), + dim=-1, + ) + + def get_pos_id_3d(self, *, t, h, w): + if t > self.max_t or h > self.max_h or w > self.max_w: + self.max_t = max(self.max_t, t) + self.max_h = max(self.max_h, h) + self.max_w = max(self.max_w, w) + self.generate_pos_id() + return self.grid[:t, :h, :w] + + +class ActionControlDiffusionDataset(Dataset): + def __init__( + self, + data_path: str | os.PathLike | None = None, + subfolder: str | None = "diffusion", + split: Literal["train", "val", "test"] = "train", + dtype: torch.dtype = torch.bfloat16, + context_seq_len: int = 512, + crossattn_embedding_size: int = 1024, + original_video_height: int = 480, + original_video_width: int = 640, + fps: int = 5, + num_frames: int = 1, + ): + """Initialize the action-control autoregressive post-training dataset. + + Args: + data_path: The path to the data. If not provided, this will assume the data is stored in the + default location in the huggingface cache. + subfolder: The subfolder to use in HF_HOME/assets/cosmos/action-control. Should not be provided + if data_path is provided. + split: The split to use. + dtype: The universal input datatype to load for the model. Modify with model layer dtype. + context_seq_len: These are predefined from the t5 text embeddings. + crossattn_embedding_size: These are the size of the hidden dim of the cross attention blocks. + original_video_height: Height dimension of the original, un-tokenized video. + original_video_width: Width dimension of the original, un-tokenized video. + fps: FPS of the video in Hz. + num_frames: Number of frames to use in each video. + """ + from cosmos1.models.autoregressive.nemo.post_training.action_control.action_control_dataset import ( + ActionControlDataset, + ) + + if subfolder is not None: + self.dataset = ActionControlDataset(subfolder=subfolder, split=split) + else: + self.dataset = ActionControlDataset(data_path=data_path, split=split) + + # Video metadata associated with the loaded dataset. + self.context_seq_len = context_seq_len + self.crossattn_embedding_size = crossattn_embedding_size + self.original_video_height = original_video_height + self.original_video_width = original_video_width + self.dtype = dtype + self.fps = torch.tensor([fps] * 1, dtype=self.dtype) + self.num_frames = torch.tensor([num_frames] * 1, dtype=self.dtype) + + def __len__(self) -> int: + """The number of valid actions in the dataset. + + Since the last frame from each trajectory can't be used as an input action, this is less + than the total number of frames. + """ + return len(self.dataset) + + def __getitem__(self, i: int) -> Dict: + """Get the i-th action-control batch from the dataset. + + Args: + i: The index of the batch to get. + + Returns: + A dictionary containing the current tokenized frame, next tokenized frame, and action. + """ + data = self.dataset[i] + # Current frame is of shape (, , height, width) + current_frame = data["current_frame"].to(self.dtype) + # Action is of shape (), which is (7) for Bridge. + action = data["action"].to(self.dtype) + # Next frame is of shape (, , height, width) + next_frame = data["next_frame"].to(self.dtype) + + # video_latent is the input to the DiT V2W model, and it accepts a tensor of shape (B, L, T=2, H, W). + # The first frame of the tensor is associated with the current video frame, i.e. the video conditioning latent, + # and the next frame of the tensor is associated with the next video frame that is predicted by the model. + # The loss is computed across both the noise of the current video frame and the predicted vs. original next frame. + video_latent = torch.cat([current_frame, next_frame], dim=-3) # concat on T dimension + # Video sequence length = T x H x W. + seq_len = video_latent.shape[-1] * video_latent.shape[-2] * video_latent.shape[-3] # W, H, T + loss_mask = torch.ones(seq_len, dtype=self.dtype) + noise_latent = torch.rand_like(video_latent, dtype=self.dtype) + timesteps = torch.randn(1, dtype=self.dtype) + # Note from Imaginaire/DIR team: we send in all zeros to our text embeddings for action control fine-tuning. + t5_text_embedding = torch.zeros((self.context_seq_len, self.crossattn_embedding_size), dtype=self.dtype) + t5_text_mask = torch.ones((self.context_seq_len), dtype=self.dtype) + image_size = torch.tensor( + [ + [ + self.original_video_height, + self.original_video_width, + self.original_video_height, + self.original_video_width, + ] + ] + * 1, + dtype=self.dtype, + ) + conditioning_latent = current_frame + padding_mask = torch.zeros((1, 1, self.original_video_height, self.original_video_width), dtype=self.dtype) + + sample = { + "video": video_latent, # tokens. We may not flatten it in the same way. AR model flattens it then + # offsets by 1 token. We may not wanna do that. + "noise_latent": noise_latent, + "timesteps": timesteps, + "t5_text_embeddings": t5_text_embedding, + "t5_text_mask": t5_text_mask, + "image_size": image_size, + "fps": self.fps, + "num_frames": self.num_frames, + "padding_mask": padding_mask, + "loss_mask": loss_mask, + "gt_latent": conditioning_latent, + "num_condition_t": 1, + "action": action, + } + + return sample + + def collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + +class VideoFolderDataset(Dataset): + def __init__(self, root_dir="", cache=True): + self.root_dir = root_dir + self.sample_prefixes = self._get_sample_prefixes() + # if cache: + # self._cache = {} + # else: + # self._cache = None + + def _get_sample_prefixes(self): + all_files = os.listdir(self.root_dir) + prefixes = set() + for file in all_files: + prefix = file.split(".")[0] + prefixes.add(prefix) + return sorted(list(prefixes)) + + def __len__(self): + return len(self.sample_prefixes) + + def __getitem__(self, idx): + # if self._cache is not None and idx in self._cache: + # return self._cache[idx] + prefix = self.sample_prefixes[idx] + + # Load JSON info + with open(os.path.join(self.root_dir, f"{prefix}.info.json"), "r") as f: + info = json.load(f) + + # Load text embeddings + text_embedding = torch.load(os.path.join(self.root_dir, f"{prefix}.t5_text_embeddings.pth")) + + # Load text mask + text_mask = torch.load(os.path.join(self.root_dir, f"{prefix}.t5_text_mask.pth")) + + # Load video latent + video_latent = torch.load(os.path.join(self.root_dir, f"{prefix}.video_latent.pth")) + + # Load conditioning latent + conditioning_latent_path = os.path.join(self.root_dir, f"{prefix}.conditioning_latent.pth") + if os.path.exists(conditioning_latent_path): + conditioning_latent = torch.load(conditioning_latent_path, map_location="cpu") + else: + conditioning_latent = None + + t = info["num_frames"] + h = info["height"] + w = info["width"] + + seq_len = video_latent.shape[-1] * video_latent.shape[-2] * video_latent.shape[-3] + loss_mask = torch.ones(seq_len, dtype=torch.bfloat16) + noise_latent = torch.rand_like(video_latent, dtype=torch.bfloat16) + timesteps = torch.randn(1) + # pos_emb = self.pos_emb_3d.get_pos_id_3d(t=t, h=h//p, w=w//p) + + sample = { + "video": video_latent, + "noise_latent": noise_latent, + "timesteps": timesteps, + "t5_text_embeddings": text_embedding, + "t5_text_mask": text_mask, + # 'pos_ids': pos_emb, + "image_size": torch.tensor([[h, w, h, w]] * 1, dtype=torch.bfloat16), + "fps": torch.tensor([info["fps"]] * 1, dtype=torch.bfloat16), + "num_frames": torch.tensor([t] * 1, dtype=torch.bfloat16), + "padding_mask": torch.zeros((1, 1, h, w), dtype=torch.bfloat16), + "loss_mask": loss_mask, + "gt_latent": conditioning_latent, + "num_condition_t": random.randint(1, 4), + } + + return sample + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +class VideoFolderCameraCtrlDataset(Dataset): + def __init__(self, root_dir="", cache=True): + self.root_dir = root_dir + self.sample_prefixes = self._get_sample_prefixes() + + def _get_sample_prefixes(self): + all_files = os.listdir(self.root_dir) + prefixes = set() + for file in all_files: + prefix = file.split(".")[0] + prefixes.add(prefix) + return sorted(list(prefixes)) + + def __len__(self): + return len(self.sample_prefixes) + + def __getitem__(self, idx): + # if self._cache is not None and idx in self._cache: + # return self._cache[idx] + prefix = self.sample_prefixes[idx] + + # Load JSON info + with open(os.path.join(self.root_dir, f"{prefix}.info.json"), "r") as f: + info = json.load(f) + + # Load text embeddings + text_embedding = torch.load(os.path.join(self.root_dir, f"{prefix}.t5_text_embeddings.pth")) + + # Load text mask + text_mask = torch.load(os.path.join(self.root_dir, f"{prefix}.t5_text_mask.pth")) + + # Load video latent + video_latent = torch.load(os.path.join(self.root_dir, f"{prefix}.video_latent.pth")) + + # Load conditioning latent + conditioning_latent_path = os.path.join(self.root_dir, f"{prefix}.conditioning_latent.pth") + conditioning_latent = torch.load(conditioning_latent_path, map_location="cpu") + + # Load plucker embeddings + plucker_embeddings_path = os.path.join(self.root_dir, f"{prefix}.plucker_embeddings.pth") + plucker_embeddings = torch.load(plucker_embeddings_path, map_location="cpu") + + # Load image size + image_size_path = os.path.join(self.root_dir, f"{prefix}.image_size.pth") + image_size = torch.load(image_size_path, map_location="cpu") + + # Load padding mask + padding_mask_path = os.path.join(self.root_dir, f"{prefix}.padding_mask.pth") + padding_mask = torch.load(padding_mask_path, map_location="cpu") + + t = info["num_frames"] + + seq_len = video_latent.shape[-1] * video_latent.shape[-2] * video_latent.shape[-3] + loss_mask = torch.ones(seq_len, dtype=torch.bfloat16) + noise_latent = torch.rand_like(video_latent, dtype=torch.bfloat16) + timesteps = torch.randn(1) + # pos_emb = self.pos_emb_3d.get_pos_id_3d(t=t, h=h//p, w=w//p) + + sample = { + "video": video_latent, + "noise_latent": noise_latent, + "timesteps": timesteps, + "t5_text_embeddings": text_embedding, + "t5_text_mask": text_mask, + # 'pos_ids': pos_emb, + "image_size": image_size, + "fps": torch.tensor([info["fps"]] * 1, dtype=torch.bfloat16), + "num_frames": torch.tensor([t] * 1, dtype=torch.bfloat16), + "padding_mask": padding_mask, + "loss_mask": loss_mask, + "gt_latent": conditioning_latent, + "num_condition_t": random.randint(1, 4), + "plucker_embeddings": plucker_embeddings, + } + + return sample + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +class DiTVideoLatentMockDataset(torch.utils.data.Dataset): + def __init__(self, num_samples, seq_len=21760): + self.length = num_samples if num_samples > 0 else 1 << 32 + self.seq_len = seq_len + self.pos_emb_3d = PosEmb3D() + + def __len__(self): + return self.length + + def __getitem__(self, idx): + t = 16 + h = 34 + w = 40 + p = 1 + seq_len = t * h * w + video_latent = torch.randn(1, 16, t, h, w).to(dtype=torch.uint8) + loss_mask = torch.ones(seq_len, dtype=torch.bfloat16) + noise_latent = torch.rand_like(video_latent, dtype=torch.bfloat16) + timesteps = torch.randn(1) + text_embedding = torch.randn(512, 1024) + pos_emb = self.pos_emb_3d.get_pos_id_3d(t=t, h=h // p, w=w // p) + + return { + "video": video_latent, + "noise_latent": noise_latent, + "timesteps": timesteps, + "t5_text_embeddings": text_embedding, + "t5_text_mask": torch.ones(512, dtype=torch.bfloat16), + "pos_ids": pos_emb, + "image_size": torch.tensor([[34, 40, 34, 40]] * 1, dtype=torch.bfloat16), + "fps": torch.tensor([30] * 1, dtype=torch.bfloat16), + "num_frames": torch.tensor([16] * 1, dtype=torch.bfloat16), + "padding_mask": torch.zeros((1, 1, 34, 40), dtype=torch.bfloat16), + "loss_mask": loss_mask, + } + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +class DiTActionDataModule(MockDataModule): + def __init__( + self, + path=None, + subfolder: str = "diffusion", + dataset=ActionControlDiffusionDataset, + dtype=torch.bfloat16, + context_seq_len: int = 512, + crossattn_embedding_size: int = 1024, + original_video_height: int = 480, + original_video_width: int = 640, + fps: int = 5, + num_frames: int = 1, + *args, + **kwargs, + ): + """ + Instantiate the datamodule. Data is automatically downloaded and cached in HF_HOME, + which can be modified in ENV. + Pass an explicit path instead of subfolder to point to an explicit dataset directory path. + """ + super().__init__(*args, **kwargs) + self.path = path + self.subfolder = subfolder + self.dataset = dataset + self.dtype = dtype + self.context_seq_len = context_seq_len + self.crossattn_embedding_size = crossattn_embedding_size + self.original_video_height = original_video_height + self.original_video_width = original_video_width + self.fps = fps + self.num_frames = num_frames + + if self.path and self.subfolder: + raise ValueError("We cannot have path and subfolder...") + + def setup(self, stage: str = "") -> None: + """ + Build ActionControlDiffusionDatasets. + """ + # Params. + params = { + "data_path": self.path, + "subfolder": self.subfolder, + "dtype": self.dtype, + "context_seq_len": self.context_seq_len, + "crossattn_embedding_size": self.crossattn_embedding_size, + "original_video_height": self.original_video_height, + "original_video_width": self.original_video_width, + "fps": self.fps, + "num_frames": self.num_frames, + } + self._train_ds = self.dataset(split="train", **params) + self._validation_ds = self.dataset(split="val", **params) + self._test_ds = self.dataset(split="test", **params) + + +class DiTDataModule(MockDataModule): + def __init__(self, *args, path="", dataset=VideoFolderDataset, **kwargs): + super().__init__(*args, **kwargs) + self.path = path + self.dataset = dataset + + def setup(self, stage: str = "") -> None: + self._train_ds = self.dataset(self.path) + self._validation_ds = self.dataset(self.path) + self._test_ds = self.dataset(self.path) + + +class DiTCameraCtrlDataModule(MockDataModule): + def __init__(self, *args, path="", dataset=VideoFolderCameraCtrlDataset, **kwargs): + super().__init__(*args, **kwargs) + self.path = path + self.dataset = dataset + + def setup(self, stage: str = "") -> None: + self._train_ds = self.dataset(self.path) + self._validation_ds = self.dataset(self.path) + self._test_ds = self.dataset(self.path) diff --git a/nemo_vfm/diffusion/encoders/__init__.py b/nemo_vfm/diffusion/encoders/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/encoders/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/encoders/conditioner.py b/nemo_vfm/diffusion/encoders/conditioner.py new file mode 100644 index 00000000..a116b4ac --- /dev/null +++ b/nemo_vfm/diffusion/encoders/conditioner.py @@ -0,0 +1,211 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import List, Optional, Union + +import torch +import torch.nn as nn +from nemo.collections.multimodal.modules.stable_diffusion.encoders.modules import LoraWrapper +from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokenizer + + +# pylint: disable=C0116 +class AbstractEmbModel(nn.Module): + def __init__( + self, + enable_lora_finetune: bool = False, + target_block: Optional[List[str]] = None, + target_module: Optional[List[str]] = None, + ) -> None: + super().__init__() + self._is_trainable = None + self._ucg_rate = None + self._input_key = None + + self.TARGET_BLOCK = target_block or [] + self.TARGET_MODULE = target_module or [] + if enable_lora_finetune: + self.lora_layers = [] + + @property + def is_trainable(self) -> bool: + return self._is_trainable + + @property + def ucg_rate(self) -> Union[float, torch.Tensor]: + return self._ucg_rate + + @property + def input_key(self) -> str: + return self._input_key + + @is_trainable.setter + def is_trainable(self, value: bool): + self._is_trainable = value + + @ucg_rate.setter + def ucg_rate(self, value: Union[float, torch.Tensor]): + self._ucg_rate = value + + @input_key.setter + def input_key(self, value: str): + self._input_key = value + + @is_trainable.deleter + def is_trainable(self): + del self._is_trainable + + @ucg_rate.deleter + def ucg_rate(self): + del self._ucg_rate + + @input_key.deleter + def input_key(self): + del self._input_key + + def encode(self, *args, **kwargs): + raise NotImplementedError + + def _enable_lora(self, lora_model): + for module_name, module in lora_model.named_modules(): + if module.__class__.__name__ in self.TARGET_BLOCK: + tmp = {} + for sub_name, sub_module in module.named_modules(): + if sub_module.__class__.__name__ in self.TARGET_MODULE: + if hasattr(sub_module, "input_size") and hasattr( + sub_module, "output_size" + ): # for megatron ParallelLinear + lora = LoraWrapper(sub_module, sub_module.input_size, sub_module.output_size) + else: # for nn.Linear + lora = LoraWrapper(sub_module, sub_module.in_features, sub_module.out_features) + self.lora_layers.append(lora) + if sub_name not in tmp.keys(): + tmp.update({sub_name: lora}) + else: + print(f"Duplicate subnames are found in module {module_name}") + for sub_name, lora_layer in tmp.items(): + lora_name = f"{sub_name}_lora" + module.add_module(lora_name, lora_layer) + + +class FrozenCLIPEmbedder(AbstractEmbModel): + """Uses the CLIP transformer encoder for text (from Hugging Face)""" + + LAYERS = ["last", "pooled", "hidden"] + + def __init__( + self, + version="openai/clip-vit-large-patch14", + device="cuda", + max_length=77, + enable_lora_finetune=False, + layer="last", + layer_idx=None, + always_return_pooled=False, + dtype=torch.float, + ): + super().__init__(enable_lora_finetune, target_block=["CLIPAttention", "CLIPMLP"], target_module=["Linear"]) + self.tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") + self.transformer = CLIPTextModel.from_pretrained(version, torch_dtype=dtype).to(device) + self.device = device + self.max_length = max_length + self.freeze() + if enable_lora_finetune: + self._enable_lora(self.transformer) + print(f"CLIP transformer encoder add {len(self.lora_layers)} lora layers.") + + self.layer = layer + self.layer_idx = layer_idx + self.return_pooled = always_return_pooled + if layer == "hidden": + assert layer_idx is not None + assert 0 <= abs(layer_idx) <= 12 + + def freeze(self): + self.transformer = self.transformer.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text, max_sequence_length=None): + batch_encoding = self.tokenizer( + text, + truncation=True, + max_length=max_sequence_length if max_sequence_length else self.max_length, + return_length=True, + return_overflowing_tokens=False, + padding="max_length", + return_tensors="pt", + ) + tokens = batch_encoding["input_ids"].to(self.transformer.device, non_blocking=True) + outputs = self.transformer(input_ids=tokens, output_hidden_states=(self.layer == "hidden")) + + if self.layer == "last": + z = outputs.last_hidden_state + elif self.layer == "pooled": + z = outputs.pooler_output[:, None, :] + else: + z = outputs.hidden_states[self.layer_idx] + + # Pad the seq length to multiple of 8 + seq_len = (z.shape[1] + 8 - 1) // 8 * 8 + z = torch.nn.functional.pad(z, (0, 0, 0, seq_len - z.shape[1]), value=0.0) + if self.return_pooled: + return z, outputs.pooler_output + return z + + def encode(self, text): + return self(text) + + +class FrozenT5Embedder(AbstractEmbModel): + def __init__( + self, + version="google/t5-v1_1-xxl", + max_length=512, + device="cuda", + dtype=torch.float, + ): + super().__init__() + self.tokenizer = T5Tokenizer.from_pretrained("google/t5-v1_1-xxl", max_length=max_length) + self.transformer = T5EncoderModel.from_pretrained(version, torch_dtype=dtype).to(device) + self.max_length = max_length + self.freeze() + self.device = device + self.dtype = dtype + + def freeze(self): + self.transformer = self.transformer.eval() + for param in self.parameters(): + param.requires_grad = False + + def forward(self, text, max_sequence_length=None): + batch_encoding = self.tokenizer( + text, + truncation=True, + max_length=max_sequence_length if max_sequence_length else self.max_length, + return_length=False, + return_overflowing_tokens=False, + padding="max_length", + return_tensors="pt", + ) + + tokens = batch_encoding["input_ids"].to(self.transformer.device, non_blocking=True) + outputs = self.transformer(input_ids=tokens, output_hidden_states=None) + + return outputs.last_hidden_state + + +# pylint: disable=C0116 diff --git a/nemo_vfm/diffusion/models/__init__.py b/nemo_vfm/diffusion/models/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/models/dit/__init__.py b/nemo_vfm/diffusion/models/dit/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/models/dit/action_control/__init__.py b/nemo_vfm/diffusion/models/dit/action_control/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/diffusion/models/dit/action_control/action_control_layers.py b/nemo_vfm/diffusion/models/dit/action_control/action_control_layers.py new file mode 100644 index 00000000..9580d2c7 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/action_control/action_control_layers.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch.nn as nn + + +class ActionControlTorchMlp(nn.Module): + def __init__( + self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0, action_3d=False + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features if not action_3d else hidden_features * 4) + self.activation = act_layer() + self.fc2 = nn.Linear( + hidden_features if not action_3d else hidden_features * 4, + out_features if not action_3d else out_features * 3, + ) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.activation(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/nemo_vfm/diffusion/models/dit/cosmos_layer_spec.py b/nemo_vfm/diffusion/models/dit/cosmos_layer_spec.py new file mode 100644 index 00000000..247ad641 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/cosmos_layer_spec.py @@ -0,0 +1,316 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import copy +from dataclasses import dataclass +from typing import Literal, Union + +import torch.nn as nn +from megatron.core.transformer.attention import ( + CrossAttention, + CrossAttentionSubmodules, + SelfAttention, + SelfAttentionSubmodules, +) +from megatron.core.transformer.custom_layers.transformer_engine import ( + TEColumnParallelLinear, + TEDotProductAttention, + TENorm, + TERowParallelLinear, +) +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_block import TransformerConfig +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from megatron.core.utils import make_viewless_tensor + + +@dataclass +class DiTWithAdaLNSubmodules(TransformerLayerSubmodules): + temporal_self_attention: Union[ModuleSpec, type] = IdentityOp + full_self_attention: Union[ModuleSpec, type] = IdentityOp + + +class AdaLN(MegatronModule): + """ + Adaptive Layer Normalization Module for DiT. + """ + + def __init__( + self, config: TransformerConfig, n_adaln_chunks=9, use_adaln_lora=True, adaln_lora_dim=256, norm=nn.LayerNorm + ): + super().__init__(config) + if norm == TENorm: + self.ln = norm(config, config.hidden_size, config.layernorm_epsilon) + else: + self.ln = norm(config.hidden_size, elementwise_affine=False, eps=self.config.layernorm_epsilon) + self.n_adaln_chunks = n_adaln_chunks + if use_adaln_lora: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(config.hidden_size, adaln_lora_dim * 3, bias=False), + nn.Linear(adaln_lora_dim * 3, self.n_adaln_chunks * config.hidden_size, bias=False), + ) + else: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(config.hidden_size, self.n_adaln_chunks * config.hidden_size, bias=False) + ) + nn.init.constant_(self.adaLN_modulation[-1].weight, 0) + + setattr(self.adaLN_modulation[-1].weight, "sequence_parallel", config.sequence_parallel) + + def forward(self, timestep_emb, adaln_lora_B_3D): + adaln_lora_B_3D = list(adaln_lora_B_3D.chunk(3, dim=-1)) + modulation = list((self.adaLN_modulation(timestep_emb)).chunk(self.n_adaln_chunks, dim=-1)) + for i in range(len(modulation)): + modulation[i] = modulation[i] + adaln_lora_B_3D[i % 3] + return modulation + + # @jit_fuser + def modulate(self, x, shift, scale): + return x * (1 + scale) + shift + + # @jit_fuser + def scale_add(self, residual, x, gate): + return residual + gate * x + + # @jit_fuser + def modulated_layernorm(self, x, shift, scale): + # Optional Input Layer norm + input_layernorm_output = self.ln(x).type_as(x) + + # z = rearrange(input_layernorm_output, '(T H W) B D -> B T H W D', T=4, H=16, W=16) + # print(f'megatron after norm = {z}, shape={z.shape}') + + # DiT block specific + return self.modulate(input_layernorm_output, shift, scale) + + # @jit_fuser + def scaled_modulated_layernorm(self, residual, x, gate, shift, scale): + hidden_states = self.scale_add(residual, x, gate) + shifted_pre_mlp_layernorm_output = self.modulated_layernorm(hidden_states, shift, scale) + return hidden_states, shifted_pre_mlp_layernorm_output + + +class DiTLayerWithAdaLN(TransformerLayer): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + + DiT with Adapative Layer Normalization. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + hidden_dropout: float = None, + position_embedding_type: Literal["learned_absolute", "rope"] = "learned_absolute", + ): + def _replace_no_cp_submodules(submodules): + modified_submods = copy.deepcopy(submodules) + modified_submods.cross_attention = IdentityOp + # modified_submods.temporal_self_attention = IdentityOp + return modified_submods + + # Replace any submodules that will have CP disabled and build them manually later after TransformerLayer init. + modified_submods = _replace_no_cp_submodules(submodules) + super().__init__( + config=config, submodules=modified_submods, layer_number=layer_number, hidden_dropout=hidden_dropout + ) + + # Override Cross Attention to disable CP. + # Disable TP Comm overlap as well. Not disabling will attempt re-use of buffer size same as + # Q and lead to incorrect tensor shapes. + if submodules.cross_attention != IdentityOp: + cp_override_config = copy.deepcopy(config) + cp_override_config.context_parallel_size = 1 + cp_override_config.tp_comm_overlap = False + self.cross_attention = build_module( + submodules.cross_attention, + config=cp_override_config, + layer_number=layer_number, + ) + else: + self.cross_attention = None + + self.full_self_attention = build_module( + submodules.full_self_attention, + config=self.config, + layer_number=layer_number, + ) + + self.adaLN = AdaLN(config=self.config, n_adaln_chunks=9 if self.cross_attention else 6) + + def forward( + self, + hidden_states, + attention_mask, + context=None, + context_mask=None, + rotary_pos_emb=None, + rotary_pos_cos=None, + rotary_pos_sin=None, + attention_bias=None, + inference_params=None, + packed_seq_params=None, + sequence_len_offset=None, + ): + rope_emb = rotary_pos_emb + extra_pos_emb = packed_seq_params["extra_pos_emb"] + adaln_lora_B_3D = packed_seq_params["adaln_lora_B_3D"] + # if type(rotary_pos_emb) == list: + # rope_emb, extra_pos_emb = rotary_pos_emb + # # extra_pos_emb = rearrange(extra_pos_emb, 'B T H W D -> (T H W) B D') + # else: + # rope_emb, extra_pos_emb = rotary_pos_emb, None + + if extra_pos_emb is not None: + hidden_states = hidden_states + extra_pos_emb + + timestep_emb = attention_mask + + # ******************************************** full self attention ******************************************* + shift_ca, scale_ca, gate_ca = None, None, None + if self.cross_attention: + shift_full, scale_full, gate_full, shift_ca, scale_ca, gate_ca, shift_mlp, scale_mlp, gate_mlp = ( + self.adaLN(timestep_emb, adaln_lora_B_3D) + ) + else: + shift_full, scale_full, gate_full, shift_mlp, scale_mlp, gate_mlp = self.adaLN( + timestep_emb, adaln_lora_B_3D + ) + + # print(f"megatron shift={shift_full}, scale={scale_full}, gate={gate_full}") + # adaLN with scale + shift + # print(f"megatron pre fa = {hidden_states}") + pre_full_attn_layernorm_output_ada = self.adaLN.modulated_layernorm( + hidden_states, + shift=shift_full, + scale=scale_full, + ) + # z = rearrange(pre_full_attn_layernorm_output_ada, '(T H W) B D -> B T H W D', T=4, H=16, W=16) + # print(f'megatron after adaLN: {z}, shape={z.shape}') + attention_output, _ = self.full_self_attention( + pre_full_attn_layernorm_output_ada, + attention_mask=None, + rotary_pos_emb=rope_emb, + # packed_seq_params=packed_seq_params['self_attention'], + ) + + # print(f'megatron fa out={attention_output}, attention_output.shape={attention_output.shape}') + + if self.cross_attention: + # ******************************************** cross attention ****************************************************** + # adaLN with scale + shift + hidden_states, pre_cross_attn_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_full, + shift=shift_ca, + scale=scale_ca, + ) + + # print(f"megatron fa out={hidden_states}") + + attention_output, _ = self.cross_attention( + pre_cross_attn_layernorm_output_ada, + attention_mask=context_mask, + key_value_states=context, + # packed_seq_params=packed_seq_params['cross_attention'], + ) + + # print(f"mcore_dit_ca_out={attention_output}") + + # ******************************************** mlp ****************************************************** + hidden_states, pre_mlp_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_ca if self.cross_attention else gate_full, + shift=shift_mlp, + scale=scale_mlp, + ) + + # print(f"megatron pre mlp={hidden_states}") + + mlp_output, _ = self.mlp(pre_mlp_layernorm_output_ada) + hidden_states = self.adaLN.scale_add(residual=hidden_states, x=mlp_output, gate=gate_mlp) + + # print(f"megatron mlp out={hidden_states}") + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + output = make_viewless_tensor(inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True) + # output = hidden_states + + return output, context + + +import transformer_engine as te + + +def get_dit_adaln_block_with_transformer_engine_spec() -> ModuleSpec: + """T5 decoder TE spec (uses Transformer Engine components).""" + # from megatron.training import get_args + # args = get_args() + # params = {"attn_mask_type": AttnMaskType.padding if + # args.packing_algorithm != 'no_packing' else AttnMaskType.no_mask} + params = {"attn_mask_type": AttnMaskType.no_mask} + return ModuleSpec( + module=DiTLayerWithAdaLN, + submodules=DiTWithAdaLNSubmodules( + full_self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=te.pytorch.RMSNorm, + k_layernorm=te.pytorch.RMSNorm, + ), + ), + cross_attention=ModuleSpec( + module=CrossAttention, + params=params, + submodules=CrossAttentionSubmodules( + linear_q=TEColumnParallelLinear, + linear_kv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=te.pytorch.RMSNorm, + k_layernorm=te.pytorch.RMSNorm, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) diff --git a/nemo_vfm/diffusion/models/dit/dit_attention.py b/nemo_vfm/diffusion/models/dit/dit_attention.py new file mode 100644 index 00000000..c0336529 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/dit_attention.py @@ -0,0 +1,460 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + +# pylint: disable=C0115,C0116,C0301 + +from dataclasses import dataclass +from typing import Union + +import torch +from megatron.core.models.common.embeddings.rotary_pos_embedding import apply_rotary_pos_emb +from megatron.core.transformer.attention import Attention, SelfAttention, SelfAttentionSubmodules +from megatron.core.transformer.custom_layers.transformer_engine import SplitAlongDim +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_config import TransformerConfig + + +@dataclass +class JointSelfAttentionSubmodules: + linear_qkv: Union[ModuleSpec, type] = None + added_linear_qkv: Union[ModuleSpec, type] = None + core_attention: Union[ModuleSpec, type] = None + linear_proj: Union[ModuleSpec, type] = None + q_layernorm: Union[ModuleSpec, type] = None + k_layernorm: Union[ModuleSpec, type] = None + added_q_layernorm: Union[ModuleSpec, type] = None + added_k_layernorm: Union[ModuleSpec, type] = None + + +# pylint: disable=C0116 +class JointSelfAttention(Attention): + """Joint Self-attention layer class + + Used for MMDIT-like transformer block. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: JointSelfAttentionSubmodules, + layer_number: int, + attn_mask_type=AttnMaskType.padding, + context_pre_only: bool = False, + ): + super().__init__( + config=config, + submodules=submodules, + layer_number=layer_number, + attn_mask_type=attn_mask_type, + attention_type="self", + ) + + self.linear_qkv = build_module( + submodules.linear_qkv, + self.config.hidden_size, + self.query_projection_size + 2 * self.kv_projection_size, + config=self.config, + init_method=self.config.init_method, + gather_output=False, + bias=self.config.add_bias_linear or self.config.add_qkv_bias, + skip_bias_add=False, + is_expert=False, + tp_comm_buffer_name="qkv", + ) + + if submodules.added_linear_qkv is not None: + self.added_linear_qkv = build_module( + submodules.added_linear_qkv, + self.config.hidden_size, + self.query_projection_size + 2 * self.kv_projection_size, + config=self.config, + init_method=self.config.init_method, + gather_output=False, + bias=self.config.add_qkv_bias, + skip_bias_add=False, + is_expert=False, + tp_comm_buffer_name="qkv", + ) + + if not context_pre_only: + self.added_linear_proj = build_module( + submodules.linear_proj, + self.query_projection_size, + self.config.hidden_size, + config=self.config, + init_method=self.config.output_layer_init_method, + bias=self.config.add_bias_linear, + input_is_parallel=True, + skip_bias_add=True, + is_expert=False, + tp_comm_buffer_name="proj", + ) + + if submodules.q_layernorm is not None: + self.q_layernorm = build_module( + submodules.q_layernorm, + hidden_size=self.hidden_size_per_attention_head, + config=self.config, + eps=self.config.layernorm_epsilon, + ) + else: + self.q_layernorm = None + + if submodules.k_layernorm is not None: + self.k_layernorm = build_module( + submodules.k_layernorm, + hidden_size=self.hidden_size_per_attention_head, + config=self.config, + eps=self.config.layernorm_epsilon, + ) + else: + self.k_layernorm = None + + if submodules.added_q_layernorm is not None: + self.added_q_layernorm = build_module( + submodules.added_q_layernorm, + hidden_size=self.hidden_size_per_attention_head, + config=self.config, + eps=self.config.layernorm_epsilon, + ) + else: + self.added_q_layernorm = None + + if submodules.added_k_layernorm is not None: + self.added_k_layernorm = build_module( + submodules.added_k_layernorm, + hidden_size=self.hidden_size_per_attention_head, + config=self.config, + eps=self.config.layernorm_epsilon, + ) + else: + self.added_k_layernorm = None + + def _split_qkv(self, mixed_qkv): + # [sq, b, hp] --> [sq, b, ng, (np/ng + 2) * hn] + new_tensor_shape = mixed_qkv.size()[:-1] + ( + self.num_query_groups_per_partition, + ( + (self.num_attention_heads_per_partition // self.num_query_groups_per_partition + 2) + * self.hidden_size_per_attention_head + ), + ) + mixed_qkv = mixed_qkv.view(*new_tensor_shape) + + split_arg_list = [ + ( + self.num_attention_heads_per_partition + // self.num_query_groups_per_partition + * self.hidden_size_per_attention_head + ), + self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + ] + + if SplitAlongDim is not None: + # [sq, b, ng, (np/ng + 2) * hn] --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn] + (query, key, value) = SplitAlongDim( + mixed_qkv, + 3, + split_arg_list, + ) + else: + # [sq, b, ng, (np/ng + 2) * hn] --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn] + (query, key, value) = torch.split( + mixed_qkv, + split_arg_list, + dim=3, + ) + + # [sq, b, ng, np/ng * hn] -> [sq, b, np, hn] + query = query.reshape(query.size(0), query.size(1), -1, self.hidden_size_per_attention_head) + return query, key, value + + def get_query_key_value_tensors(self, hidden_states, key_value_states=None): + """ + Derives `query`, `key` and `value` tensors from `hidden_states`. + """ + # Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)] + mixed_qkv, _ = self.linear_qkv(hidden_states) + + query, key, value = self._split_qkv(mixed_qkv) + + if self.config.test_mode: + self.run_realtime_tests() + + if self.q_layernorm is not None: + query = self.q_layernorm(query) + + if self.k_layernorm is not None: + key = self.k_layernorm(key) + + return query, key, value + + def get_added_query_key_value_tensors(self, added_hidden_states, key_value_states=None): + """ + Derives `query`, `key` and `value` tensors from `hidden_states`. + """ + # Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)] + mixed_qkv, _ = self.added_linear_qkv(added_hidden_states) + + query, key, value = self._split_qkv(mixed_qkv) + + if self.config.test_mode: + self.run_realtime_tests() + + if self.added_q_layernorm is not None: + query = self.added_q_layernorm(query) + + if self.added_k_layernorm is not None: + key = self.added_k_layernorm(key) + + return query, key, value + + def forward( + self, + hidden_states, + attention_mask, + key_value_states=None, + inference_params=None, + rotary_pos_emb=None, + packed_seq_params=None, + additional_hidden_states=None, + ): + # hidden_states: [sq, b, h] + # For self attention we just duplicate the rotary_pos_emb if it isn't already + if rotary_pos_emb is not None and not isinstance(rotary_pos_emb, tuple): + rotary_pos_emb = (rotary_pos_emb,) * 2 + # ===================== + # Query, Key, and Value + # ===================== + # Get the query, key and value tensors based on the type of attention - + # self or cross attn. + + query, key, value = self.get_query_key_value_tensors(hidden_states) + added_query, added_key, added_value = self.get_added_query_key_value_tensors(additional_hidden_states) + + query = torch.cat([added_query, query], dim=0) + key = torch.cat([added_key, key], dim=0) + value = torch.cat([added_value, value], dim=0) + + # =================================================== + # Adjust key, value, and rotary_pos_emb for inference + # =================================================== + query, key, value, rotary_pos_emb, attn_mask_type = self._adjust_key_value_for_inference( + inference_params, query, key, value, rotary_pos_emb + ) + + if packed_seq_params is not None: + query = query.squeeze(1) + key = key.squeeze(1) + value = value.squeeze(1) + + # ================================================ + # relative positional embedding (rotary embedding) + # ================================================ + if rotary_pos_emb is not None: + q_pos_emb, k_pos_emb = rotary_pos_emb + + if packed_seq_params is not None: + cu_seqlens_q = packed_seq_params.cu_seqlens_q + cu_seqlens_kv = packed_seq_params.cu_seqlens_kv + else: + cu_seqlens_q = cu_seqlens_kv = None + query = apply_rotary_pos_emb( + query, + q_pos_emb, + config=self.config, + cu_seqlens=cu_seqlens_q, + ) + key = apply_rotary_pos_emb( + key, + k_pos_emb, + config=self.config, + cu_seqlens=cu_seqlens_kv, + ) + + # TODO, can apply positional embedding to value_layer so it has + # absolute positional embedding. + # otherwise, only relative positional embedding takes effect + # value_layer = apply_rotary_pos_emb(value_layer, k_pos_emb) + + # ================================== + # core attention computation + # ================================== + if self.checkpoint_core_attention and self.training: + core_attn_out = self._checkpointed_attention_forward( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + packed_seq_params=packed_seq_params, + ) + else: + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + packed_seq_params=packed_seq_params, + ) + + if packed_seq_params is not None: + # reshape to same output shape as unpacked case + # (t, np, hn) -> (t, b=1, h=np*hn) + # t is the pack size = sum (sq_i) + # note that batch is a dummy dimension in the packed case + core_attn_out = core_attn_out.reshape(core_attn_out.size(0), 1, -1) + + # ================= + # Output. [sq, b, h] + # ================= + encoder_attention_output = core_attn_out[: additional_hidden_states.shape[0], :, :] + attention_output = core_attn_out[additional_hidden_states.shape[0] :, :, :] + + output, bias = self.linear_proj(attention_output) + encoder_output, encoder_bias = self.added_linear_proj(encoder_attention_output) + + output = output + bias + encoder_output = encoder_output + encoder_bias + + return output, encoder_output + + +class FluxSingleAttention(SelfAttention): + """Self-attention layer class + + Self-attention layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: SelfAttentionSubmodules, + layer_number: int, + attn_mask_type=AttnMaskType.padding, + cp_comm_type: str = None, + ): + super().__init__( + config=config, + submodules=submodules, + layer_number=layer_number, + attn_mask_type=attn_mask_type, + cp_comm_type=cp_comm_type, + ) + self.linear_proj = build_module( + submodules.linear_proj, + self.query_projection_size, + self.config.hidden_size, + config=self.config, + init_method=self.config.output_layer_init_method, + bias=False, + input_is_parallel=True, + skip_bias_add=True, + is_expert=False, + tp_comm_buffer_name="proj", + ) + + def forward( + self, + hidden_states, + attention_mask, + key_value_states=None, + inference_params=None, + rotary_pos_emb=None, + packed_seq_params=None, + ): + # hidden_states: [sq, b, h] + + # For self attention we just duplicate the rotary_pos_emb if it isn't already + if rotary_pos_emb is not None and not isinstance(rotary_pos_emb, tuple): + rotary_pos_emb = (rotary_pos_emb,) * 2 + + # ===================== + # Query, Key, and Value + # ===================== + # Get the query, key and value tensors based on the type of attention - + # self or cross attn. + query, key, value = self.get_query_key_value_tensors(hidden_states, key_value_states) + # print(f'megatron q before ln: {query.transpose(0, 1).contiguous()}, {query.transpose(0, 1).contiguous().shape}') + # print(f'megatron k before ln: {key.transpose(0, 1).contiguous()}, {key.transpose(0, 1).contiguous().shape}') + # print(f'megatron v before ln: {value.transpose(0, 1).contiguous()}, {value.transpose(0, 1).contiguous().shape}') + + # =================================================== + # Adjust key, value, and rotary_pos_emb for inference + # =================================================== + query, key, value, rotary_pos_emb, attn_mask_type = self._adjust_key_value_for_inference( + inference_params, query, key, value, rotary_pos_emb + ) + + if packed_seq_params is not None: + query = query.squeeze(1) + key = key.squeeze(1) + value = value.squeeze(1) + + # ================================================ + # relative positional embedding (rotary embedding) + # ================================================ + if rotary_pos_emb is not None: + q_pos_emb, k_pos_emb = rotary_pos_emb + + if packed_seq_params is not None: + cu_seqlens_q = packed_seq_params.cu_seqlens_q + cu_seqlens_kv = packed_seq_params.cu_seqlens_kv + else: + cu_seqlens_q = cu_seqlens_kv = None + query = apply_rotary_pos_emb( + query, + q_pos_emb, + config=self.config, + cu_seqlens=cu_seqlens_q, + ) + key = apply_rotary_pos_emb( + key, + k_pos_emb, + config=self.config, + cu_seqlens=cu_seqlens_kv, + ) + + # TODO, can apply positional embedding to value_layer so it has + # absolute positional embedding. + # otherwise, only relative positional embedding takes effect + # value_layer = apply_rotary_pos_emb(value_layer, k_pos_emb) + + # ================================== + # core attention computation + # ================================== + + if self.checkpoint_core_attention and self.training: + core_attn_out = self._checkpointed_attention_forward( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + packed_seq_params=packed_seq_params, + ) + else: + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + packed_seq_params=packed_seq_params, + ) + + if packed_seq_params is not None: + # reshape to same output shape as unpacked case + # (t, np, hn) -> (t, b=1, h=np*hn) + # t is the pack size = sum (sq_i) + # note that batch is a dummy dimension in the packed case + core_attn_out = core_attn_out.reshape(core_attn_out.size(0), 1, -1) + + output, _ = self.linear_proj(core_attn_out) + return output + + +# pylint: disable=C0116 diff --git a/nemo_vfm/diffusion/models/dit/dit_embeddings.py b/nemo_vfm/diffusion/models/dit/dit_embeddings.py new file mode 100644 index 00000000..5bbfd5db --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/dit_embeddings.py @@ -0,0 +1,247 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + + +import logging +from typing import Optional + +import torch +from diffusers.models.embeddings import TimestepEmbedding, get_3d_sincos_pos_embed +from einops import rearrange +from megatron.core import parallel_state +from megatron.core.transformer.module import MegatronModule +from torch import nn + + +log = logging.getLogger(__name__) + + +class SDXLTimestepEmbedding(nn.Module): + def __init__(self, in_features: int, out_features: int, use_adaln_lora: bool = False): + super().__init__() + log.critical( + f"Using AdaLN LoRA Flag: {use_adaln_lora}. We enable bias if no AdaLN LoRA for backward compatibility." + ) + self.linear_1 = nn.Linear(in_features, out_features, bias=not use_adaln_lora) + self.activation = nn.SiLU() + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.linear_2 = nn.Linear(out_features, 3 * out_features, bias=False) + else: + self.linear_2 = nn.Linear(out_features, out_features, bias=True) + + def forward(self, sample: torch.Tensor) -> torch.Tensor: + emb = self.linear_1(sample) + emb = self.activation(emb) + emb = self.linear_2(emb) + + if self.use_adaln_lora: + adaln_lora_B_3D = emb + emb_B_D = sample + else: + emb_B_D = emb + adaln_lora_B_3D = None + + return emb_B_D, adaln_lora_B_3D + + +class ParallelSDXLTimestepEmbedding(SDXLTimestepEmbedding): + def __init__( + self, + in_features: int, + out_features: int, + use_adaln_lora: bool = False, + seed: Optional[int] = None, + ): + super().__init__( + in_features=in_features, + out_features=out_features, + use_adaln_lora=use_adaln_lora, + ) + if seed is not None: + with torch.random.fork_rng(): + torch.manual_seed(seed) + self.linear_1.reset_parameters() + self.linear_2.reset_parameters() + + # Check for pipeline model parallelism and set attributes accordingly + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + setattr(self.linear_1.weight, "pipeline_parallel", True) + if self.linear_1.bias is not None: + setattr(self.linear_1.bias, "pipeline_parallel", True) + setattr(self.linear_2.weight, "pipeline_parallel", True) + if self.linear_2.bias is not None: + setattr(self.linear_2.bias, "pipeline_parallel", True) + + def forward(self, sample: torch.Tensor) -> torch.Tensor: + sample = sample.to(torch.bfloat16, non_blocking=True) + return super().forward(sample) + + +class ParallelTimestepEmbedding(TimestepEmbedding): + """ + ParallelTimestepEmbedding is a subclass of TimestepEmbedding that initializes + the embedding layers with an optional random seed for syncronization. + + Args: + in_channels (int): Number of input channels. + time_embed_dim (int): Dimension of the time embedding. + seed (int, optional): Random seed for initializing the embedding layers. + If None, no specific seed is set. + + Attributes: + linear_1 (nn.Module): First linear layer for the embedding. + linear_2 (nn.Module): Second linear layer for the embedding. + + Methods: + __init__(in_channels, time_embed_dim, seed=None): Initializes the embedding layers. + """ + + def __init__(self, in_channels: int, time_embed_dim: int, seed=None): + super().__init__(in_channels=in_channels, time_embed_dim=time_embed_dim) + if seed is not None: + with torch.random.fork_rng(): + torch.manual_seed(seed) + self.linear_1.reset_parameters() + self.linear_2.reset_parameters() + + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + setattr(self.linear_1.weight, "pipeline_parallel", True) + setattr(self.linear_1.bias, "pipeline_parallel", True) + setattr(self.linear_2.weight, "pipeline_parallel", True) + setattr(self.linear_2.bias, "pipeline_parallel", True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Computes the positional embeddings for the input tensor. + + Args: + x (torch.Tensor): Input tensor of shape (B, T, H, W, C). + + Returns: + torch.Tensor: Positional embeddings of shape (B, T, H, W, C). + """ + return super().forward(x.to(torch.bfloat16, non_blocking=True)) + + +def get_pos_emb_on_this_cp_rank(pos_emb, seq_dim): + """ + Adjusts the positional embeddings tensor to the current context parallel rank. + + Args: + pos_emb (torch.Tensor): The positional embeddings tensor. + seq_dim (int): The sequence dimension index in the positional embeddings tensor. + + Returns: + torch.Tensor: The adjusted positional embeddings tensor for the current context parallel rank. + """ + cp_size = parallel_state.get_context_parallel_world_size() + cp_rank = parallel_state.get_context_parallel_rank() + cp_idx = torch.tensor([cp_rank], device="cpu", pin_memory=True).cuda(non_blocking=True) + pos_emb = pos_emb.view(*pos_emb.shape[:seq_dim], cp_size, -1, *pos_emb.shape[(seq_dim + 1) :]) + pos_emb = pos_emb.index_select(seq_dim, cp_idx) + pos_emb = pos_emb.view(*pos_emb.shape[:seq_dim], -1, *pos_emb.shape[(seq_dim + 2) :]) + return pos_emb + + +class SinCosPosEmb3D(MegatronModule): + """ + SinCosPosEmb3D is a 3D sine-cosine positional embedding module. + + Args: + model_channels (int): Number of channels in the model. + h (int): Length of the height dimension. + w (int): Length of the width dimension. + t (int): Length of the temporal dimension. + spatial_interpolation_scale (float, optional): Scale factor for spatial interpolation. Default is 1.0. + temporal_interpolation_scale (float, optional): Scale factor for temporal interpolation. Default is 1.0. + + Methods: + forward(pos_ids: torch.Tensor) -> torch.Tensor: + Computes the positional embeddings for the input tensor. + + Args: + pos_ids (torch.Tensor): Input tensor of shape (B S 3). + + Returns: + torch.Tensor: Positional embeddings of shape (B S D). + """ + + def __init__( + self, + config, + h: int, + w: int, + t: int, + spatial_interpolation_scale=1.0, + temporal_interpolation_scale=1.0, + ): + super().__init__(config=config) + self.h = h + self.w = w + self.t = t + # h w t + param = get_3d_sincos_pos_embed( + config.hidden_size, [h, w], t, spatial_interpolation_scale, temporal_interpolation_scale + ) + param = rearrange(param, "t hw c -> (t hw) c") + self.pos_embedding = torch.nn.Embedding(param.shape[0], config.hidden_size) + self.pos_embedding.weight = torch.nn.Parameter(torch.tensor(param), requires_grad=False) + + def forward(self, pos_ids: torch.Tensor): + # pos_ids: t h w + pos_id = pos_ids[..., 0] * self.h * self.w + pos_ids[..., 1] * self.w + pos_ids[..., 2] + return self.pos_embedding(pos_id) + + +class FactorizedLearnable3DEmbedding(MegatronModule): + def __init__( + self, + config, + t: int, + h: int, + w: int, + **kwargs, + ): + super().__init__(config=config) + self.emb_t = torch.nn.Embedding(t, config.hidden_size) + self.emb_h = torch.nn.Embedding(h, config.hidden_size) + self.emb_w = torch.nn.Embedding(w, config.hidden_size) + + if "seed" in kwargs.keys(): + seed = kwargs["seed"] + with torch.random.fork_rng(): + torch.manual_seed(seed) + if config.perform_initialization: + self.customize_init_param() + else: + self.reset_parameters() + else: + if config.perform_initialization: + self.customize_init_param() + + def customize_init_param(self): + self.config.init_method(self.emb_t.weight) + self.config.init_method(self.emb_h.weight) + self.config.init_method(self.emb_w.weight) + + def reset_parameters(self): + self.emb_t.reset_parameters() + self.emb_h.reset_parameters() + self.emb_w.reset_parameters() + + def forward(self, pos_ids: torch.Tensor): + return self.emb_t(pos_ids[..., 0]) + self.emb_h(pos_ids[..., 1]) + self.emb_w(pos_ids[..., 2]) diff --git a/nemo_vfm/diffusion/models/dit/dit_layer_spec.py b/nemo_vfm/diffusion/models/dit/dit_layer_spec.py new file mode 100644 index 00000000..2ebd5998 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/dit_layer_spec.py @@ -0,0 +1,893 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import copy +from dataclasses import dataclass +from typing import Literal, Union + +import torch +import torch.nn as nn +from megatron.core.jit import jit_fuser +from megatron.core.transformer.attention import ( + CrossAttention, + CrossAttentionSubmodules, + SelfAttention, + SelfAttentionSubmodules, +) +from megatron.core.transformer.cuda_graphs import CudaGraphManager +from megatron.core.transformer.custom_layers.transformer_engine import ( + TEColumnParallelLinear, + TEDotProductAttention, + TENorm, + TERowParallelLinear, +) +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_block import TransformerConfig +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from megatron.core.utils import make_viewless_tensor +from nemo.collections.diffusion.models.dit.dit_attention import ( + FluxSingleAttention, + JointSelfAttention, + JointSelfAttentionSubmodules, +) + + +# pylint: disable=C0116 +@dataclass +class DiTWithAdaLNSubmodules(TransformerLayerSubmodules): + temporal_self_attention: Union[ModuleSpec, type] = IdentityOp + full_self_attention: Union[ModuleSpec, type] = IdentityOp + + +@dataclass +class STDiTWithAdaLNSubmodules(TransformerLayerSubmodules): + spatial_self_attention: Union[ModuleSpec, type] = IdentityOp + temporal_self_attention: Union[ModuleSpec, type] = IdentityOp + full_self_attention: Union[ModuleSpec, type] = IdentityOp + + +class RMSNorm(nn.Module): + def __init__(self, hidden_size: int, config, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + return output * self.weight + + +class AdaLN(MegatronModule): + """ + Adaptive Layer Normalization Module for DiT. + """ + + def __init__( + self, config: TransformerConfig, n_adaln_chunks=9, use_adaln_lora=True, adaln_lora_dim=256, norm=nn.LayerNorm + ): + super().__init__(config) + if norm == TENorm: + self.ln = norm(config, config.hidden_size, config.layernorm_epsilon) + else: + self.ln = norm(config.hidden_size, elementwise_affine=False, eps=self.config.layernorm_epsilon) + self.n_adaln_chunks = n_adaln_chunks + if use_adaln_lora: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(config.hidden_size, adaln_lora_dim, bias=False), + nn.Linear(adaln_lora_dim, self.n_adaln_chunks * config.hidden_size, bias=False), + ) + else: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(config.hidden_size, self.n_adaln_chunks * config.hidden_size, bias=False) + ) + nn.init.constant_(self.adaLN_modulation[-1].weight, 0) + + setattr(self.adaLN_modulation[-1].weight, "sequence_parallel", config.sequence_parallel) + + def forward(self, timestep_emb): + return self.adaLN_modulation(timestep_emb).chunk(self.n_adaln_chunks, dim=-1) + + @jit_fuser + def modulate(self, x, shift, scale): + return x * (1 + scale) + shift + + @jit_fuser + def scale_add(self, residual, x, gate): + return residual + gate * x + + @jit_fuser + def modulated_layernorm(self, x, shift, scale): + # Optional Input Layer norm + input_layernorm_output = self.ln(x).type_as(x) + + # DiT block specific + return self.modulate(input_layernorm_output, shift, scale) + + # @jit_fuser + def scaled_modulated_layernorm(self, residual, x, gate, shift, scale): + hidden_states = self.scale_add(residual, x, gate) + shifted_pre_mlp_layernorm_output = self.modulated_layernorm(hidden_states, shift, scale) + return hidden_states, shifted_pre_mlp_layernorm_output + + +class AdaLNContinuous(MegatronModule): + """ + A variant of AdaLN used for flux models. + """ + + def __init__( + self, + config: TransformerConfig, + conditioning_embedding_dim: int, + modulation_bias: bool = True, + norm_type: str = "layer_norm", + ): + super().__init__(config) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(conditioning_embedding_dim, config.hidden_size * 2, bias=modulation_bias) + ) + if norm_type == "layer_norm": + self.norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False, eps=1e-6, bias=modulation_bias) + elif norm_type == "rms_norm": + self.norm = RMSNorm(config.hidden_size, eps=1e-6) + else: + raise ValueError("Unknown normalization type {}".format(norm_type)) + + def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor: + emb = self.adaLN_modulation(conditioning_embedding) + scale, shift = torch.chunk(emb, 2, dim=1) + x = self.norm(x) * (1 + scale) + shift + return x + + +class STDiTLayerWithAdaLN(TransformerLayer): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + + Spatial-Temporal DiT with Adapative Layer Normalization. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + hidden_dropout: float = None, + position_embedding_type: Literal["learned_absolute", "rope"] = "learned_absolute", + ): + def _replace_no_cp_submodules(submodules): + modified_submods = copy.deepcopy(submodules) + modified_submods.cross_attention = IdentityOp + modified_submods.spatial_self_attention = IdentityOp + return modified_submods + + # Replace any submodules that will have CP disabled and build them manually later after TransformerLayer init. + modified_submods = _replace_no_cp_submodules(submodules) + super().__init__( + config=config, submodules=modified_submods, layer_number=layer_number, hidden_dropout=hidden_dropout + ) + + # Override Spatial Self Attention and Cross Attention to disable CP. + # Disable TP Comm overlap as well. Not disabling will attempt re-use of buffer size same as Q and lead to + # incorrect tensor shapes. + sa_cp_override_config = copy.deepcopy(config) + sa_cp_override_config.context_parallel_size = 1 + sa_cp_override_config.tp_comm_overlap = False + self.spatial_self_attention = build_module( + submodules.spatial_self_attention, config=sa_cp_override_config, layer_number=layer_number + ) + self.cross_attention = build_module( + submodules.cross_attention, + config=sa_cp_override_config, + layer_number=layer_number, + ) + + self.temporal_self_attention = build_module( + submodules.temporal_self_attention, + config=self.config, + layer_number=layer_number, + ) + + self.full_self_attention = build_module( + submodules.full_self_attention, + config=self.config, + layer_number=layer_number, + ) + + self.adaLN = AdaLN(config=self.config, n_adaln_chunks=3) + + def forward( + self, + hidden_states, + attention_mask, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + ): + # timestep embedding + timestep_emb = attention_mask + + # ******************************************** spatial self attention ***************************************** + + shift_sa, scale_sa, gate_sa = self.adaLN(timestep_emb) + + # adaLN with scale + shift + pre_spatial_attn_layernorm_output_ada = self.adaLN.modulated_layernorm( + hidden_states, shift=shift_sa, scale=scale_sa + ) + + attention_output, _ = self.spatial_self_attention( + pre_spatial_attn_layernorm_output_ada, + attention_mask=None, + # packed_seq_params=packed_seq_params['self_attention'], + ) + + # ******************************************** full self attention ******************************************** + + shift_full, scale_full, gate_full = self.adaLN(timestep_emb) + + # adaLN with scale + shift + hidden_states, pre_full_attn_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_sa, + shift=shift_full, + scale=scale_full, + ) + + attention_output, _ = self.full_self_attention( + pre_full_attn_layernorm_output_ada, + attention_mask=None, + # packed_seq_params=packed_seq_params['self_attention'], + ) + + # ******************************************** cross attention ************************************************ + + shift_ca, scale_ca, gate_ca = self.adaLN(timestep_emb) + + # adaLN with scale + shift + hidden_states, pre_cross_attn_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_full, + shift=shift_ca, + scale=scale_ca, + ) + + attention_output, _ = self.cross_attention( + pre_cross_attn_layernorm_output_ada, + attention_mask=context_mask, + key_value_states=context, + # packed_seq_params=packed_seq_params['cross_attention'], + ) + + # ******************************************** temporal self attention **************************************** + + shift_ta, scale_ta, gate_ta = self.adaLN(timestep_emb) + + hidden_states, pre_temporal_attn_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_ca, + shift=shift_ta, + scale=scale_ta, + ) + + attention_output, _ = self.temporal_self_attention( + pre_temporal_attn_layernorm_output_ada, + attention_mask=None, + # packed_seq_params=packed_seq_params['self_attention'], + ) + + # ******************************************** mlp ************************************************************ + + shift_mlp, scale_mlp, gate_mlp = self.adaLN(timestep_emb) + + hidden_states, pre_mlp_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_ta, + shift=shift_mlp, + scale=scale_mlp, + ) + + mlp_output, _ = self.mlp(pre_mlp_layernorm_output_ada) + hidden_states = self.adaLN.scale_add(residual=hidden_states, x=mlp_output, gate=gate_mlp) + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + output = make_viewless_tensor(inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True) + + return output, context + + +class DiTLayerWithAdaLN(TransformerLayer): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + + DiT with Adapative Layer Normalization. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + hidden_dropout: float = None, + position_embedding_type: Literal["learned_absolute", "rope"] = "learned_absolute", + ): + def _replace_no_cp_submodules(submodules): + modified_submods = copy.deepcopy(submodules) + modified_submods.cross_attention = IdentityOp + # modified_submods.temporal_self_attention = IdentityOp + return modified_submods + + # Replace any submodules that will have CP disabled and build them manually later after TransformerLayer init. + modified_submods = _replace_no_cp_submodules(submodules) + super().__init__( + config=config, submodules=modified_submods, layer_number=layer_number, hidden_dropout=hidden_dropout + ) + + # Override Cross Attention to disable CP. + # Disable TP Comm overlap as well. Not disabling will attempt re-use of buffer size same as Q and lead to + # incorrect tensor shapes. + if submodules.cross_attention != IdentityOp: + cp_override_config = copy.deepcopy(config) + cp_override_config.context_parallel_size = 1 + cp_override_config.tp_comm_overlap = False + self.cross_attention = build_module( + submodules.cross_attention, + config=cp_override_config, + layer_number=layer_number, + ) + else: + self.cross_attention = None + + self.full_self_attention = build_module( + submodules.full_self_attention, + config=self.config, + layer_number=layer_number, + ) + + self.adaLN = AdaLN(config=self.config, n_adaln_chunks=9 if self.cross_attention else 6) + + def forward( + self, + hidden_states, + attention_mask, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + ): + # timestep embedding + timestep_emb = attention_mask + + # ******************************************** full self attention ******************************************** + if self.cross_attention: + shift_full, scale_full, gate_full, shift_ca, scale_ca, gate_ca, shift_mlp, scale_mlp, gate_mlp = ( + self.adaLN(timestep_emb) + ) + else: + shift_full, scale_full, gate_full, shift_mlp, scale_mlp, gate_mlp = self.adaLN(timestep_emb) + + # adaLN with scale + shift + pre_full_attn_layernorm_output_ada = self.adaLN.modulated_layernorm( + hidden_states, shift=shift_full, scale=scale_full + ) + + attention_output, _ = self.full_self_attention( + pre_full_attn_layernorm_output_ada, + attention_mask=None, + packed_seq_params=None if packed_seq_params is None else packed_seq_params["self_attention"], + ) + + if self.cross_attention: + # ******************************************** cross attention ******************************************** + # adaLN with scale + shift + hidden_states, pre_cross_attn_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_full, + shift=shift_ca, + scale=scale_ca, + ) + + attention_output, _ = self.cross_attention( + pre_cross_attn_layernorm_output_ada, + attention_mask=context_mask, + key_value_states=context, + packed_seq_params=None if packed_seq_params is None else packed_seq_params["cross_attention"], + ) + + # ******************************************** mlp ****************************************************** + hidden_states, pre_mlp_layernorm_output_ada = self.adaLN.scaled_modulated_layernorm( + residual=hidden_states, + x=attention_output, + gate=gate_ca if self.cross_attention else gate_full, + shift=shift_mlp, + scale=scale_mlp, + ) + + mlp_output, _ = self.mlp(pre_mlp_layernorm_output_ada) + hidden_states = self.adaLN.scale_add(residual=hidden_states, x=mlp_output, gate=gate_mlp) + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + output = make_viewless_tensor(inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True) + + return output, context + + +class DiTLayer(TransformerLayer): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + + Original DiT layer implementation from [https://arxiv.org/pdf/2212.09748]. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + mlp_ratio: int = 4, + n_adaln_chunks: int = 6, + modulation_bias: bool = True, + ): + # Modify the mlp layer hidden_size of a dit layer according to mlp_ratio + config.ffn_hidden_size = int(mlp_ratio * config.hidden_size) + super().__init__(config=config, submodules=submodules, layer_number=layer_number) + + self.adaLN = AdaLN( + config=config, n_adaln_chunks=n_adaln_chunks, modulation_bias=modulation_bias, use_second_norm=True + ) + + def forward( + self, + hidden_states, + attention_mask, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + ): + # passing in conditioning information via attention mask here + c = attention_mask + + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN(c) + + shifted_input_layernorm_output = self.adaLN.modulated_layernorm( + hidden_states, shift=shift_msa, scale=scale_msa, layernorm_idx=0 + ) + + x, bias = self.self_attention(shifted_input_layernorm_output, attention_mask=None) + + hidden_states = self.adaLN.scale_add(hidden_states, x=(x + bias), gate=gate_msa) + + residual = hidden_states + + shited_pre_mlp_layernorm_output = self.adaLN.modulated_layernorm( + hidden_states, shift=shift_mlp, scale=scale_mlp, layernorm_idx=1 + ) + + x, bias = self.mlp(shited_pre_mlp_layernorm_output) + + hidden_states = self.adaLN.scale_add(residual, x=(x + bias), gate=gate_mlp) + + return hidden_states, context + + +class MMDiTLayer(TransformerLayer): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + + MMDiT layer implementation from [https://arxiv.org/pdf/2403.03206]. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + context_pre_only: bool = False, + ): + hidden_size = config.hidden_size + super().__init__(config=config, submodules=submodules, layer_number=layer_number) + + if config.enable_cuda_graph: + self.cudagraph_manager = CudaGraphManager(config, share_cudagraph_io_buffers=False) + + self.adaln = AdaLN(config, modulation_bias=True, n_adaln_chunks=6, use_second_norm=True) + + self.context_pre_only = context_pre_only + context_norm_type = "ada_norm_continuous" if context_pre_only else "ada_norm_zero" + + if context_norm_type == "ada_norm_continuous": + self.adaln_context = AdaLNContinuous(config, hidden_size, modulation_bias=True, norm_type="layer_norm") + elif context_norm_type == "ada_norm_zero": + self.adaln_context = AdaLN(config, modulation_bias=True, n_adaln_chunks=6, use_second_norm=True) + else: + raise ValueError( + f"Unknown context_norm_type: {context_norm_type}, " + f"currently only support `ada_norm_continous`, `ada_norm_zero`" + ) + # Override Cross Attention to disable CP. + # Disable TP Comm overlap as well. Not disabling will attempt re-use of buffer size same as Q and lead to + # incorrect tensor shapes. + cp_override_config = copy.deepcopy(config) + cp_override_config.context_parallel_size = 1 + cp_override_config.tp_comm_overlap = False + + if not context_pre_only: + self.context_mlp = build_module( + submodules.mlp, + config=cp_override_config, + ) + else: + self.context_mlp = None + + def forward( + self, + hidden_states, + encoder_hidden_states, + attention_mask=None, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + emb=None, + ): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln(emb) + + norm_hidden_states = self.adaln.modulated_layernorm( + hidden_states, shift=shift_msa, scale=scale_msa, layernorm_idx=0 + ) + if self.context_pre_only: + norm_encoder_hidden_states = self.adaln_context(encoder_hidden_states, emb) + else: + c_shift_msa, c_scale_msa, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.adaln_context(emb) + norm_encoder_hidden_states = self.adaln_context.modulated_layernorm( + encoder_hidden_states, shift=c_shift_msa, scale=c_scale_msa, layernorm_idx=0 + ) + + attention_output, encoder_attention_output = self.self_attention( + norm_hidden_states, + attention_mask=attention_mask, + key_value_states=None, + additional_hidden_states=norm_encoder_hidden_states, + rotary_pos_emb=rotary_pos_emb, + ) + hidden_states = self.adaln.scale_add(hidden_states, x=attention_output, gate=gate_msa) + norm_hidden_states = self.adaln.modulated_layernorm( + hidden_states, shift=shift_mlp, scale=scale_mlp, layernorm_idx=1 + ) + + mlp_output, mlp_output_bias = self.mlp(norm_hidden_states) + hidden_states = self.adaln.scale_add(hidden_states, x=(mlp_output + mlp_output_bias), gate=gate_mlp) + + if self.context_pre_only: + encoder_hidden_states = None + else: + encoder_hidden_states = self.adaln_context.scale_add( + encoder_hidden_states, x=encoder_attention_output, gate=c_gate_msa + ) + norm_encoder_hidden_states = self.adaln_context.modulated_layernorm( + encoder_hidden_states, shift=c_shift_mlp, scale=c_scale_mlp, layernorm_idx=1 + ) + + context_mlp_output, context_mlp_output_bias = self.context_mlp(norm_encoder_hidden_states) + encoder_hidden_states = self.adaln.scale_add( + encoder_hidden_states, x=(context_mlp_output + context_mlp_output_bias), gate=c_gate_mlp + ) + + return hidden_states, encoder_hidden_states + + def __call__(self, *args, **kwargs): + if hasattr(self, "cudagraph_manager"): + return self.cudagraph_manager(self, args, kwargs) + return super(MegatronModule, self).__call__(*args, **kwargs) + + +class FluxSingleTransformerBlock(TransformerLayer): + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + mlp_ratio: int = 4, + n_adaln_chunks: int = 3, + modulation_bias: bool = True, + ): + super().__init__(config=config, submodules=submodules, layer_number=layer_number) + + if config.enable_cuda_graph: + self.cudagraph_manager = CudaGraphManager(config, share_cudagraph_io_buffers=False) + self.adaln = AdaLN( + config=config, n_adaln_chunks=n_adaln_chunks, modulation_bias=modulation_bias, use_second_norm=False + ) + + def forward( + self, + hidden_states, + attention_mask=None, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + emb=None, + ): + residual = hidden_states + + shift, scale, gate = self.adaln(emb) + + norm_hidden_states = self.adaln.modulated_layernorm(hidden_states, shift=shift, scale=scale) + + mlp_hidden_states, mlp_bias = self.mlp(norm_hidden_states) + + attention_output = self.self_attention( + norm_hidden_states, attention_mask=attention_mask, rotary_pos_emb=rotary_pos_emb + ) + + hidden_states = mlp_hidden_states + mlp_bias + attention_output + + hidden_states = self.adaln.scale_add(residual, x=hidden_states, gate=gate) + + return hidden_states, None + + def __call__(self, *args, **kwargs): + if hasattr(self, "cudagraph_manager"): + return self.cudagraph_manager(self, args, kwargs) + return super(MegatronModule, self).__call__(*args, **kwargs) + + +def get_stdit_adaln_block_with_transformer_engine_spec() -> ModuleSpec: + params = {"attn_mask_type": AttnMaskType.padding} + return ModuleSpec( + module=STDiTLayerWithAdaLN, + submodules=STDiTWithAdaLNSubmodules( + spatial_self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=TENorm, + k_layernorm=TENorm, + ), + ), + temporal_self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=TENorm, + k_layernorm=TENorm, + ), + ), + full_self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=TENorm, + k_layernorm=TENorm, + ), + ), + cross_attention=ModuleSpec( + module=CrossAttention, + params=params, + submodules=CrossAttentionSubmodules( + linear_q=TEColumnParallelLinear, + linear_kv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=TENorm, + k_layernorm=TENorm, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) + + +def get_dit_adaln_block_with_transformer_engine_spec() -> ModuleSpec: + params = {"attn_mask_type": AttnMaskType.padding} + return ModuleSpec( + module=DiTLayerWithAdaLN, + submodules=DiTWithAdaLNSubmodules( + full_self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=RMSNorm, + k_layernorm=RMSNorm, + ), + ), + cross_attention=ModuleSpec( + module=CrossAttention, + params=params, + submodules=CrossAttentionSubmodules( + linear_q=TEColumnParallelLinear, + linear_kv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=RMSNorm, + k_layernorm=RMSNorm, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) + + +def get_official_dit_adaln_block_with_transformer_engine_spec() -> ModuleSpec: + params = {"attn_mask_type": AttnMaskType.no_mask} + return ModuleSpec( + module=DiTLayerWithAdaLN, + submodules=DiTWithAdaLNSubmodules( + full_self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) + + +def get_mm_dit_block_with_transformer_engine_spec() -> ModuleSpec: + return ModuleSpec( + module=MMDiTLayer, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=JointSelfAttention, + params={"attn_mask_type": AttnMaskType.no_mask}, + submodules=JointSelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + added_linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) + + +def get_flux_single_transformer_engine_spec() -> ModuleSpec: + return ModuleSpec( + module=FluxSingleTransformerBlock, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=FluxSingleAttention, + params={"attn_mask_type": AttnMaskType.no_mask}, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + q_layernorm=RMSNorm, + k_layernorm=RMSNorm, + linear_proj=TERowParallelLinear, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) + + +def get_flux_double_transformer_engine_spec() -> ModuleSpec: + return ModuleSpec( + module=MMDiTLayer, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=JointSelfAttention, + params={"attn_mask_type": AttnMaskType.no_mask}, + submodules=JointSelfAttentionSubmodules( + q_layernorm=RMSNorm, + k_layernorm=RMSNorm, + added_q_layernorm=RMSNorm, + added_k_layernorm=RMSNorm, + linear_qkv=TEColumnParallelLinear, + added_linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) + + +# pylint: disable=C0116 diff --git a/nemo_vfm/diffusion/models/dit/dit_model.py b/nemo_vfm/diffusion/models/dit/dit_model.py new file mode 100644 index 00000000..6f99297c --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/dit_model.py @@ -0,0 +1,375 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Dict, Literal, Optional + +import torch +import torch.distributed +import torch.nn as nn +from diffusers.models.embeddings import Timesteps +from einops import rearrange, repeat +from megatron.core import parallel_state, tensor_parallel +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.enums import ModelType +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.utils import make_sharded_tensor_for_checkpoint +from nemo.collections.diffusion.models.dit import dit_embeddings +from nemo.collections.diffusion.models.dit.dit_embeddings import ParallelTimestepEmbedding +from nemo.collections.diffusion.models.dit.dit_layer_spec import ( + get_dit_adaln_block_with_transformer_engine_spec as DiTLayerWithAdaLNspec, +) +from torch import Tensor + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class RMSNorm(nn.Module): + def __init__(self, channel: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(channel)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + return output * self.weight + + +class FinalLayer(nn.Module): + """ + The final layer of DiT. + """ + + def __init__(self, hidden_size, spatial_patch_size, temporal_patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear( + hidden_size, spatial_patch_size * spatial_patch_size * temporal_patch_size * out_channels, bias=False + ) + self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=False)) + + def forward(self, x_BT_HW_D, emb_B_D): + shift_B_D, scale_B_D = self.adaLN_modulation(emb_B_D).chunk(2, dim=1) + T = x_BT_HW_D.shape[0] // emb_B_D.shape[0] + shift_BT_D, scale_BT_D = repeat(shift_B_D, "b d -> (b t) d", t=T), repeat(scale_B_D, "b d -> (b t) d", t=T) + x_BT_HW_D = modulate(self.norm_final(x_BT_HW_D), shift_BT_D, scale_BT_D) + x_BT_HW_D = self.linear(x_BT_HW_D) + return x_BT_HW_D + + +class DiTCrossAttentionModel(VisionModule): + """ + DiTCrossAttentionModel is a VisionModule that implements a DiT model with a cross-attention block. + Attributes: + config (TransformerConfig): Configuration for the transformer. + pre_process (bool): Whether to apply pre-processing steps. + post_process (bool): Whether to apply post-processing steps. + fp16_lm_cross_entropy (bool): Whether to use fp16 for cross-entropy loss. + parallel_output (bool): Whether to use parallel output. + position_embedding_type (Literal["learned_absolute", "rope"]): Type of position embedding. + max_img_h (int): Maximum image height. + max_img_w (int): Maximum image width. + max_frames (int): Maximum number of frames. + patch_spatial (int): Spatial patch size. + patch_temporal (int): Temporal patch size. + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + transformer_decoder_layer_spec (DiTLayerWithAdaLNspec): Specification for the transformer decoder layer. + add_encoder (bool): Whether to add an encoder. + add_decoder (bool): Whether to add a decoder. + share_embeddings_and_output_weights (bool): Whether to share embeddings and output weights. + concat_padding_mask (bool): Whether to concatenate padding mask. + pos_emb_cls (str): Class of position embedding. + model_type (ModelType): Type of the model. + decoder (TransformerBlock): Transformer decoder block. + t_embedder (torch.nn.Sequential): Time embedding layer. + x_embedder (nn.Conv3d): Convolutional layer for input embedding. + pos_embedder (dit_embeddings.SinCosPosEmb3D): Position embedding layer. + final_layer_linear (torch.nn.Linear): Final linear layer. + affline_norm (RMSNorm): Affine normalization layer. + Methods: + forward(x: Tensor, timesteps: Tensor, crossattn_emb: Tensor, packed_seq_params: PackedSeqParams = None, pos_ids: Tensor = None, **kwargs) -> Tensor: + Forward pass of the model. + set_input_tensor(input_tensor: Tensor) -> None: + Sets input tensor to the model. + sharded_state_dict(prefix: str = 'module.', sharded_offsets: tuple = (), metadata: Optional[Dict] = None) -> ShardedStateDict: + Sharded state dict implementation for backward-compatibility. + tie_embeddings_weights_state_dict(tensor, sharded_state_dict: ShardedStateDict, output_layer_weight_key: str, first_stage_word_emb_key: str) -> None: + Ties the embedding and output weights in a given sharded state dict. + """ + + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + max_img_h: int = 80, + max_img_w: int = 80, + max_frames: int = 34, + patch_spatial: int = 1, + patch_temporal: int = 1, + in_channels: int = 16, + out_channels: int = 16, + transformer_decoder_layer_spec=DiTLayerWithAdaLNspec, + pos_embedder=dit_embeddings.SinCosPosEmb3D, + **kwargs, + ): + super(DiTCrossAttentionModel, self).__init__(config=config) + + self.config: TransformerConfig = config + + self.transformer_decoder_layer_spec = transformer_decoder_layer_spec() + self.pre_process = pre_process + self.post_process = post_process + self.add_encoder = True + self.add_decoder = True + self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.parallel_output = parallel_output + self.position_embedding_type = position_embedding_type + self.share_embeddings_and_output_weights = False + self.concat_padding_mask = True + self.pos_emb_cls = "sincos" + self.patch_spatial = patch_spatial + self.patch_temporal = patch_temporal + + # megatron core pipelining currently depends on model type + # TODO: remove this dependency ? + self.model_type = ModelType.encoder_or_decoder + + # Transformer decoder + self.decoder = TransformerBlock( + config=self.config, + spec=self.transformer_decoder_layer_spec, + pre_process=self.pre_process, + post_process=False, + post_layer_norm=False, + ) + + self.t_embedder = torch.nn.Sequential( + Timesteps(self.config.hidden_size, flip_sin_to_cos=False, downscale_freq_shift=0), + dit_embeddings.ParallelTimestepEmbedding(self.config.hidden_size, self.config.hidden_size, seed=1234), + ) + + self.fps_embedder = nn.Sequential( + Timesteps(num_channels=256, flip_sin_to_cos=False, downscale_freq_shift=1), + ParallelTimestepEmbedding(256, 256, seed=1234), + ) + + if self.pre_process: + self.x_embedder = torch.nn.Linear(in_channels * patch_spatial**2, self.config.hidden_size) + + if pos_embedder is dit_embeddings.SinCosPosEmb3D: + if self.pre_process: + self.pos_embedder = pos_embedder( + config, + t=max_frames // patch_temporal, + h=max_img_h // patch_spatial, + w=max_img_w // patch_spatial, + ) + else: + # here I just follow the original logic, that except with SinCosPosEmb3D, the pos_emb would be feeded to transformer blocks, + # so the other embedders should be replicated across pp ranks. + self.pos_embedder = pos_embedder( + config, + t=max_frames // patch_temporal, + h=max_img_h // patch_spatial, + w=max_img_w // patch_spatial, + seed=1234, + ) + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + for p in self.pos_embedder.parameters(): + setattr(p, "pipeline_parallel", True) + + if self.post_process: + self.final_layer_linear = torch.nn.Linear( + self.config.hidden_size, + patch_spatial**2 * patch_temporal * out_channels, + ) + + self.affline_norm = RMSNorm(self.config.hidden_size) + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + setattr(self.affline_norm.weight, "pipeline_parallel", True) + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded data (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + B = x.shape[0] + fps = kwargs.get( + "fps", + torch.tensor( + [ + 30, + ] + * B, + dtype=torch.bfloat16, + ), + ).view(-1) + if self.pre_process: + # transpose to match + x_B_S_D = self.x_embedder(x) + if isinstance(self.pos_embedder, dit_embeddings.SinCosPosEmb3D): + pos_emb = None + x_B_S_D += self.pos_embedder(pos_ids) + else: + pos_emb = self.pos_embedder(pos_ids) + pos_emb = rearrange(pos_emb, "B S D -> S B D") + x_S_B_D = rearrange(x_B_S_D, "B S D -> S B D") + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + if (not hasattr(self, "pos_embedder")) or isinstance(self.pos_embedder, dit_embeddings.SinCosPosEmb3D): + pos_emb = None + else: + # if transformer blocks need pos_emb, then pos_embedder should + # be replicated across pp ranks. + pos_emb = rearrange(self.pos_embedder(pos_ids), "B S D -> S B D") + + timesteps_B_D = self.t_embedder(timesteps.flatten()).to(torch.bfloat16) # (b d_text_embedding) + + affline_emb_B_D = timesteps_B_D + fps_B_D = self.fps_embedder(fps) + fps_B_D = nn.functional.pad(fps_B_D, (0, self.config.hidden_size - fps_B_D.shape[1])) + affline_emb_B_D += fps_B_D + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if isinstance(self.pos_embedder, dit_embeddings.FactorizedLearnable3DEmbedding): + pos_emb = tensor_parallel.scatter_to_sequence_parallel_region(pos_emb) + + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + crossattn_emb = crossattn_emb.clone() + + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + rotary_pos_emb=pos_emb, + packed_seq_params=packed_seq_params, + ) + + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_S_B_D = self.final_layer_linear(x_S_B_D) + return rearrange(x_S_B_D, "S B D -> B S D") + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Sets input tensor to the model. + + See megatron.model.transformer.set_input_tensor() + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + assert len(input_tensor) == 1, "input_tensor should only be length 1 for gpt/bert" + self.decoder.set_input_tensor(input_tensor[0]) + + def sharded_state_dict( + self, prefix: str = "module.", sharded_offsets: tuple = (), metadata: Optional[Dict] = None + ) -> ShardedStateDict: + """Sharded state dict implementation for GPTModel backward-compatibility (removing extra state). + + Args: + prefix (str): Module name prefix. + sharded_offsets (tuple): PP related offsets, expected to be empty at this module level. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. + + Returns: + ShardedStateDict: sharded state dict for the GPTModel + """ + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) + + for module in ["t_embedder"]: + for param_name, param in getattr(self, module).named_parameters(): + weight_key = f"{prefix}{module}.{param_name}" + self._set_embedder_weights_replica_id(param, sharded_state_dict, weight_key) + return sharded_state_dict + + def _set_embedder_weights_replica_id( + self, tensor: Tensor, sharded_state_dict: ShardedStateDict, embedder_weight_key: str + ) -> None: + """set replica ids of the weights in t_embedder for sharded state dict. + + Args: + sharded_state_dict (ShardedStateDict): state dict with the weight to tie + weight_key (str): key of the weight in the state dict. + This entry will be replaced with a tied version + + Returns: None, acts in-place + """ + tp_rank = parallel_state.get_tensor_model_parallel_rank() + vpp_rank = parallel_state.get_virtual_pipeline_model_parallel_rank() + vpp_rank = vpp_rank if vpp_rank else 0 + vpp_world = parallel_state.get_virtual_pipeline_model_parallel_world_size() + vpp_world = vpp_world if vpp_world else 1 + pp_rank = parallel_state.get_pipeline_model_parallel_rank() + del sharded_state_dict[embedder_weight_key] + replica_id = ( + tp_rank, + (vpp_rank + pp_rank * vpp_world), + parallel_state.get_data_parallel_rank(with_context_parallel=True), + ) + + sharded_state_dict[embedder_weight_key] = make_sharded_tensor_for_checkpoint( + tensor=tensor, + key=embedder_weight_key, + replica_id=replica_id, + allow_shape_mismatch=False, + ) diff --git a/nemo_vfm/diffusion/models/dit/dit_model_14b.py b/nemo_vfm/diffusion/models/dit/dit_model_14b.py new file mode 100644 index 00000000..f465f6a3 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/dit_model_14b.py @@ -0,0 +1,1709 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +import warnings +from typing import Dict, List, Literal, Optional, Tuple + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from megatron.core import InferenceParams, parallel_state, tensor_parallel +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.utils import make_sharded_tensor_for_checkpoint +from nemo.collections.diffusion.models.dit.action_control.action_control_layers import ActionControlTorchMlp +from nemo.collections.diffusion.models.dit.cosmos_layer_spec import ( + get_dit_adaln_block_with_transformer_engine_spec as DiTLayerWithAdaLNspec, +) +from torch import Tensor +from torch.autograd import Function +from torch.distributed import ProcessGroup, get_process_group_ranks +from torchvision import transforms + + +def gather_along_first_dim(tensor, process_group): + return AllGather.apply(tensor, process_group) + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class SDXLTimesteps(nn.Module): + def __init__(self, num_channels: int = 320): + super().__init__() + self.num_channels = num_channels + + def forward(self, timesteps): + in_dype = timesteps.dtype + half_dim = self.num_channels // 2 + exponent = -math.log(10000) * torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) + exponent = exponent / (half_dim - 0.0) + + emb = torch.exp(exponent) + emb = timesteps[:, None].float() * emb[None, :] + + sin_emb = torch.sin(emb) + cos_emb = torch.cos(emb) + emb = torch.cat([cos_emb, sin_emb], dim=-1) + + return emb.to(in_dype) + + +class SDXLTimestepEmbedding(nn.Module): + def __init__(self, in_features: int, out_features: int, use_adaln_lora: bool = False): + super().__init__() + self.linear_1 = nn.Linear(in_features, out_features, bias=not use_adaln_lora) + self.activation = nn.SiLU() + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.linear_2 = nn.Linear(out_features, 3 * out_features, bias=False) + else: + self.linear_2 = nn.Linear(out_features, out_features, bias=True) + + def forward(self, sample: torch.Tensor) -> torch.Tensor: + emb = self.linear_1(sample) + emb = self.activation(emb) + emb = self.linear_2(emb) + + if self.use_adaln_lora: + adaln_lora_B_3D = emb + emb_B_D = sample + else: + emb_B_D = emb + adaln_lora_B_3D = None + + return emb_B_D, adaln_lora_B_3D + + +class AllGather(Function): + @staticmethod + def forward(ctx, tensor, process_group): + world_size = dist.get_world_size(process_group) + ctx.world_size = world_size + ctx.rank = process_group.rank() + + gathered_tensors = [torch.zeros_like(tensor) for _ in range(world_size)] + dist.all_gather(gathered_tensors, tensor.contiguous(), process_group) + return torch.cat(gathered_tensors, dim=0) + + @staticmethod + def backward(ctx, grad_output): + world_size = ctx.world_size + rank = ctx.rank + + # Split the gradient tensor + grad_chunks = grad_output.chunk(world_size) + + # Select the gradient chunk for the current rank + grad_input = grad_chunks[rank] + return grad_input, None + + +class PatchEmbed(nn.Module): + # TODO: (qsh 2024-09-20) update docstring + """ + PatchEmbed is a module for embedding patches from an input tensor by applying either 3D or 2D convolutional layers, + depending on the . This module can process inputs with temporal (video) and spatial (image) dimensions, + making it suitable for video and image processing tasks. It supports dividing the input into patches and embedding each + patch into a vector of size `out_channels`. + + Parameters: + - spatial_patch_size (int): The size of each spatial patch. + - temporal_patch_size (int): The size of each temporal patch. + - in_channels (int): Number of input channels. Default: 3. + - out_channels (int): The dimension of the embedding vector for each patch. Default: 768. + - bias (bool): If True, adds a learnable bias to the output of the convolutional layers. Default: True. + - keep_spatio (bool): If True, the spatial dimensions are kept separate in the output tensor, otherwise, they are flattened. Default: False. + - legacy_patch_emb (bool): If True, applies 3D convolutional layers for video inputs, otherwise, use Linear! The legacy model is for backward compatibility. Default: True. + The output shape of the module depends on the `keep_spatio` flag. If `keep_spatio`=True, the output retains the spatial dimensions. + Otherwise, the spatial dimensions are flattened into a single dimension. + """ + + def __init__( + self, + spatial_patch_size, + temporal_patch_size, + in_channels=3, + out_channels=768, + bias=True, + keep_spatio=False, + legacy_patch_emb: bool = True, + ): + super().__init__() + self.spatial_patch_size = spatial_patch_size + self.temporal_patch_size = temporal_patch_size + assert keep_spatio, "Only support keep_spatio=True" + self.keep_spatio = keep_spatio + self.legacy_patch_emb = legacy_patch_emb + + if legacy_patch_emb: + self.proj = nn.Conv3d( + in_channels, + out_channels, + kernel_size=(temporal_patch_size, spatial_patch_size, spatial_patch_size), + stride=(temporal_patch_size, spatial_patch_size, spatial_patch_size), + bias=bias, + ) + self.out = Rearrange("b c t h w -> b t h w c") + else: + self.proj = nn.Sequential( + Rearrange( + "b c (t r) (h m) (w n) -> b t h w (c r m n)", + r=temporal_patch_size, + m=spatial_patch_size, + n=spatial_patch_size, + ), + nn.Linear( + in_channels * spatial_patch_size * spatial_patch_size * temporal_patch_size, + out_channels, + bias=bias, + ), + ) + self.out = nn.Identity() + + def forward(self, x): + """ + Forward pass of the PatchEmbed module. + + Parameters: + - x (torch.Tensor): The input tensor of shape (B, C, T, H, W) where + B is the batch size, + C is the number of channels, + T is the temporal dimension, + H is the height, and + W is the width of the input. + + Returns: + - torch.Tensor: The embedded patches as a tensor, with shape b t h w c. + """ + assert x.dim() == 5 + _, _, T, H, W = x.shape + assert H % self.spatial_patch_size == 0 and W % self.spatial_patch_size == 0 + assert T % self.temporal_patch_size == 0 + x = self.proj(x) + return self.out(x) + + +class FourierFeatures(nn.Module): + """ + Implements a layer that generates Fourier features from input tensors, based on randomly sampled + frequencies and phases. This can help in learning high-frequency functions in low-dimensional problems. + + [B] -> [B, D] + + Parameters: + num_channels (int): The number of Fourier features to generate. + bandwidth (float, optional): The scaling factor for the frequency of the Fourier features. Defaults to 1. + normalize (bool, optional): If set to True, the outputs are scaled by sqrt(2), usually to normalize + the variance of the features. Defaults to False. + + Example: + >>> layer = FourierFeatures(num_channels=256, bandwidth=0.5, normalize=True) + >>> x = torch.randn(10, 256) # Example input tensor + >>> output = layer(x) + >>> print(output.shape) # Expected shape: (10, 256) + """ + + def __init__(self, num_channels, bandwidth=1, normalize=False): + super().__init__() + self.register_buffer("freqs", 2 * np.pi * bandwidth * torch.randn(num_channels), persistent=True) + self.register_buffer("phases", 2 * np.pi * torch.rand(num_channels), persistent=True) + self.gain = np.sqrt(2) if normalize else 1 + + def forward(self, x, gain: float = 1.0): + """ + Apply the Fourier feature transformation to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + gain (float, optional): An additional gain factor applied during the forward pass. Defaults to 1. + + Returns: + torch.Tensor: The transformed tensor, with Fourier features applied. + """ + in_dtype = x.dtype + x = x.to(torch.float32).ger(self.freqs.to(torch.float32)).add(self.phases.to(torch.float32)) + x = x.cos().mul(self.gain * gain).to(in_dtype) + return x + + +class FinalLayer(nn.Module): + """ + The final layer of video DiT. + """ + + def __init__( + self, + hidden_size, + spatial_patch_size, + temporal_patch_size, + out_channels, + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + ): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear( + hidden_size, spatial_patch_size * spatial_patch_size * temporal_patch_size * out_channels, bias=False + ) + self.hidden_size = hidden_size + self.n_adaln_chunks = 2 + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, adaln_lora_dim, bias=False), + nn.Linear(adaln_lora_dim, self.n_adaln_chunks * hidden_size, bias=False), + ) + else: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, self.n_adaln_chunks * hidden_size, bias=False) + ) + + self.sequence_parallel = getattr(parallel_state, "sequence_parallel", False) + + def forward( + self, + x_BT_HW_D, + emb_B_D, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ): + if self.use_adaln_lora: + assert adaln_lora_B_3D is not None + shift_B_D, scale_B_D = (self.adaLN_modulation(emb_B_D) + adaln_lora_B_3D[:, : 2 * self.hidden_size]).chunk( + 2, dim=1 + ) + else: + shift_B_D, scale_B_D = self.adaLN_modulation(emb_B_D).chunk(2, dim=1) + + B = emb_B_D.shape[0] + T = x_BT_HW_D.shape[0] // B + shift_BT_D, scale_BT_D = repeat(shift_B_D, "b d -> (b t) d", t=T), repeat(scale_B_D, "b d -> (b t) d", t=T) + x_BT_HW_D = modulate(self.norm_final(x_BT_HW_D), shift_BT_D, scale_BT_D) + if self.sequence_parallel: + x_T_B_HW_D = rearrange(x_BT_HW_D, "(b t) hw d -> t b hw d", b=B, t=T) + x_T_B_HW_D = gather_along_first_dim(x_T_B_HW_D, parallel_state.get_tensor_model_parallel_group()) + x_BT_HW_D = rearrange(x_T_B_HW_D, "t b hw d -> (b t) hw d", b=B) + + x_BT_HW_D = self.linear(x_BT_HW_D) + return x_BT_HW_D + + +class RMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + return output * self.weight + + +def split_inputs_cp(x: Tensor, seq_dim: int, cp_group: ProcessGroup) -> Tensor: + """ + Split input tensor along the sequence dimension for checkpoint parallelism. + + This function divides the input tensor into equal parts along the specified + sequence dimension, based on the number of ranks in the checkpoint parallelism group. + It then selects the part corresponding to the current rank. + + Args: + x: Input tensor to be split. + seq_dim: The dimension along which to split the input (sequence dimension). + cp_group: The process group for checkpoint parallelism. + + Returns: + A slice of the input tensor corresponding to the current rank. + + Raises: + AssertionError: If the sequence dimension is not divisible by the number of ranks. + """ + cp_ranks = get_process_group_ranks(cp_group) + cp_size = len(cp_ranks) + + assert x.shape[seq_dim] % cp_size == 0, f"{x.shape[seq_dim]} cannot divide cp_size {cp_size}" + x = x.view(*x.shape[:seq_dim], cp_size, x.shape[seq_dim] // cp_size, *x.shape[(seq_dim + 1) :]) + seq_idx = torch.tensor([cp_group.rank()], device=x.device) + x = x.index_select(seq_dim, seq_idx) + # Note that the new sequence length is the original sequence length / cp_size + x = x.view(*x.shape[:seq_dim], -1, *x.shape[(seq_dim + 2) :]) + return x + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +def normalize(x: torch.Tensor, dim: Optional[List[int]] = None, eps: float = 0) -> torch.Tensor: + """ + Normalizes the input tensor along specified dimensions such that the average square norm of elements is adjusted. + + Args: + x (torch.Tensor): The input tensor to normalize. + dim (list, optional): The dimensions over which to normalize. If None, normalizes over all dimensions except the first. + eps (float, optional): A small constant to ensure numerical stability during division. + + Returns: + torch.Tensor: The normalized tensor. + """ + if dim is None: + dim = list(range(1, x.ndim)) + norm = torch.linalg.vector_norm(x, dim=dim, keepdim=True, dtype=torch.float32) + norm = torch.add(eps, norm, alpha=np.sqrt(norm.numel() / x.numel())) + return x / norm.to(x.dtype) + + +class VideoPositionEmb(nn.Module): + def __init__(self): + super().__init__() + self.cp_group = None + + def enable_context_parallel(self, cp_group: ProcessGroup): + self.cp_group = cp_group + + def disable_context_parallel(self): + self.cp_group = None + + def forward(self, x_B_T_H_W_C: torch.Tensor, fps=Optional[torch.Tensor]) -> torch.Tensor: + """ + With CP, the function assume that the input tensor is already split. It delegates the embedding generation to generate_embeddings function. + """ + B_T_H_W_C = x_B_T_H_W_C.shape + if self.cp_group is not None: + cp_ranks = get_process_group_ranks(self.cp_group) + cp_size = len(cp_ranks) + B, T, H, W, C = B_T_H_W_C + B_T_H_W_C = (B, T * cp_size, H, W, C) + embeddings = self.generate_embeddings(B_T_H_W_C, fps=fps) + + if self.cp_group is not None: + if isinstance(self, VideoRopePosition3DEmb): + seq_dim = 0 + else: + seq_dim = 1 + embeddings = split_inputs_cp(x=embeddings, seq_dim=seq_dim, cp_group=self.cp_group) + return embeddings + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]): + raise NotImplementedError + + +class VideoRopePosition3DEmb(VideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + head_dim: int, + len_h: int, + len_w: int, + len_t: int, + base_fps: int = 24, + h_extrp_ratio: float = 1.0, + w_extrp_ratio: float = 1.0, + t_extrp_ratio: float = 1.0, + **kwargs, # used for compatibility with other positional embeddings; unused in this class + ): + del kwargs + super().__init__() + self.register_buffer("seq", torch.arange(max(len_h, len_w, len_t), dtype=torch.float)) + self.base_fps = base_fps + self.max_h = len_h + self.max_w = len_w + + dim = head_dim + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + self.register_buffer( + "dim_spatial_range", + torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().cuda() / dim_h, + persistent=False, + ) + self.register_buffer( + "dim_temporal_range", + torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().cuda() / dim_t, + persistent=False, + ) + + self.h_ntk_factor = h_extrp_ratio ** (dim_h / (dim_h - 2)) + self.w_ntk_factor = w_extrp_ratio ** (dim_w / (dim_w - 2)) + self.t_ntk_factor = t_extrp_ratio ** (dim_t / (dim_t - 2)) + + def generate_embeddings( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. Defaults to None. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. Defaults to None. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. Defaults to None. + + Returns: + Not specified in the original code snippet. + """ + h_ntk_factor = h_ntk_factor if h_ntk_factor is not None else self.h_ntk_factor + w_ntk_factor = w_ntk_factor if w_ntk_factor is not None else self.w_ntk_factor + t_ntk_factor = t_ntk_factor if t_ntk_factor is not None else self.t_ntk_factor + + h_theta = 10000.0 * h_ntk_factor + w_theta = 10000.0 * w_ntk_factor + t_theta = 10000.0 * t_ntk_factor + + h_spatial_freqs = 1.0 / (h_theta**self.dim_spatial_range) + w_spatial_freqs = 1.0 / (w_theta**self.dim_spatial_range) + temporal_freqs = 1.0 / (t_theta**self.dim_temporal_range) + + B, T, H, W, _ = B_T_H_W_C + assert B == 1 or T == 1, ( + "Batch size should be 1 or T should be 1. Image batch should have T=1, while video batch should have B=1." + ) + assert H <= self.max_h and W <= self.max_w, ( + f"Input dimensions (H={H}, W={W}) exceed the maximum dimensions (max_h={self.max_h}, max_w={self.max_w}) configured for positional embedding. Please adjust the input size or increase the maximum dimensions in the model configuration." + ) + self.seq = self.seq.cuda() + half_emb_h = torch.outer(self.seq[:H], h_spatial_freqs) + half_emb_w = torch.outer(self.seq[:W], w_spatial_freqs) + + # apply sequence scaling in temporal dimension + if fps is None: # image case + assert T == 1, "T should be 1 for image batch." + half_emb_t = torch.outer(self.seq[:T], temporal_freqs) + else: + half_emb_t = torch.outer(self.seq[:T] / fps[:1] * self.base_fps, temporal_freqs) + + em_T_H_W_D = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + + return rearrange(em_T_H_W_D, "t h w d -> (t h w) 1 1 d").float() + + +class SinCosPosEmbAxis(VideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + interpolation: str, + model_channels: int, + len_h: int, + len_w: int, + len_t: int, + is_learnable: bool = True, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + **kwargs, + ): + # TODO: (qsh 2024-11-08) add more interpolation methods and args for extrapolation fine-tuning + """ + Args: + interpolation (str): we curretly only support "crop", ideally when we need extrapolation capacity, we should adjust frequency or other more advanced methods. they are not implemented yet. + """ + del kwargs # unused + super().__init__() + self.interpolation = interpolation + self.is_learnable = is_learnable + assert self.interpolation in ["crop"], f"Unknown interpolation method {self.interpolation}" + + self.pos_emb_h = nn.Parameter(torch.zeros(len_h, model_channels)) + self.pos_emb_w = nn.Parameter(torch.zeros(len_w, model_channels)) + self.pos_emb_t = nn.Parameter(torch.zeros(len_t, model_channels)) + + trunc_normal_(self.pos_emb_h, std=0.02) + trunc_normal_(self.pos_emb_w, std=0.02) + trunc_normal_(self.pos_emb_t, std=0.02) + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]) -> torch.Tensor: + B, T, H, W, C = B_T_H_W_C + if self.interpolation == "crop": + emb_h_H = self.pos_emb_h[:H] + emb_w_W = self.pos_emb_w[:W] + emb_t_T = self.pos_emb_t[:T] + emb = ( + repeat(emb_t_T, "t d-> b t h w d", b=B, h=H, w=W) + + repeat(emb_h_H, "h d-> b t h w d", b=B, t=T, w=W) + + repeat(emb_w_W, "w d-> b t h w d", b=B, t=T, h=H) + ) + assert list(emb.shape)[:4] == [B, T, H, W], f"bad shape: {list(emb.shape)[:4]} != {B, T, H, W}" + else: + raise ValueError(f"Unknown interpolation method {self.interpolation}") + + return normalize(emb, dim=-1, eps=1e-6) + + +class DiTCrossAttentionModel14B(VisionModule): + """DiT with CrossAttention model. + + Args: + config (TransformerConfig): transformer config + + transformer_decoder_layer_spec (ModuleSpec): transformer layer customization specs for decoder + + pre_process (bool): Include embedding layer (used with pipeline parallelism) + post_process (bool): Include an output layer (used with pipeline parallelism) + + fp16_lm_cross_entropy (bool, optional): Defaults to False + + parallel_output (bool): Do not gather the outputs, keep them split across tensor parallel ranks + + share_embeddings_and_output_weights (bool): When True, input embeddings and output logit weights are + shared. Defaults to False. + + position_embedding_type (string): Position embedding type. Options ['learned_absolute', 'rope']. + Defaults is 'learned_absolute'. + + rotary_percent (float): Percent of rotary dimension to use for rotary position embeddings. + Defaults to 1.0 (100%). Ignored unless position_embedding_type is 'rope'. + + seq_len_interpolation_factor (float): scale of linearly interpolating RoPE for longer sequences. + The value must be a float larger than 1.0. Defaults to None. + """ + + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + rotary_percent: float = 1.0, + seq_len_interpolation_factor: Optional[float] = None, + ): + super(DiTCrossAttentionModel14B, self).__init__(config=config) + + self.config: TransformerConfig = config + from megatron.core.enums import ModelType + + self.model_type = ModelType.encoder_or_decoder + + self.transformer_decoder_layer_spec = DiTLayerWithAdaLNspec() + self.pre_process = pre_process + self.post_process = post_process + self.add_encoder = True + self.add_decoder = True + self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.parallel_output = parallel_output + self.position_embedding_type = position_embedding_type + self.share_embeddings_and_output_weights = False + self.additional_timestamp_channels = False + self.concat_padding_mask = True + self.pos_emb_cls = "rope3d" + self.use_adaln_lora = True + self.patch_spatial = 2 + self.patch_temporal = 1 + self.out_channels = 16 + self.adaln_lora_dim = 256 + + # Transformer decoder + self.decoder = TransformerBlock( + config=self.config, + spec=self.transformer_decoder_layer_spec, + pre_process=self.pre_process, + post_process=self.post_process, + post_layer_norm=False, + ) + + self.t_embedder = nn.Sequential( + SDXLTimesteps(self.config.hidden_size), + SDXLTimestepEmbedding( + self.config.hidden_size, self.config.hidden_size, use_adaln_lora=self.use_adaln_lora + ), + ) + + if self.pre_process: + self.in_channels = 16 + self.in_channels = self.in_channels + 1 if self.concat_padding_mask else self.in_channels + self.legacy_patch_emb = False + self.x_embedder = ( + PatchEmbed( + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + in_channels=self.in_channels, + out_channels=self.config.hidden_size, + bias=False, + keep_spatio=True, + legacy_patch_emb=self.legacy_patch_emb, + ) + .cuda() + .to(dtype=torch.bfloat16) + ) + + self.max_img_h = 240 + self.max_img_w = 240 + self.max_frames = 128 + self.min_fps = 1 + self.max_fps = 30 + self.pos_emb_learnable = False + self.pos_emb_interpolation = "crop" + self.rope_h_extrp_ratio = 2.0 + self.rope_w_extrp_ratio = 2.0 + self.rope_t_extrp_ratio = 2.0 + self.extra_per_block_abs_pos_emb = True + + self.pos_embedder = VideoRopePosition3DEmb( + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + max_fps=self.max_fps, + min_fps=self.min_fps, + is_learnable=self.pos_emb_learnable, + interpolation=self.pos_emb_interpolation, + head_dim=self.config.hidden_size // self.config.num_attention_heads, + h_extrp_ratio=self.rope_h_extrp_ratio, + w_extrp_ratio=self.rope_w_extrp_ratio, + t_extrp_ratio=self.rope_t_extrp_ratio, + ) + + if self.extra_per_block_abs_pos_emb: + self.extra_pos_embedder = SinCosPosEmbAxis( + h_extrapolation_ratio=2, + w_extrapolation_ratio=2, + t_extrapolation_ratio=2, + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + interpolation=self.pos_emb_interpolation, + ) + + if parallel_state.get_context_parallel_world_size() > 1: + cp_group = parallel_state.get_context_parallel_group() + self.pos_embedder.enable_context_parallel(cp_group) + self.extra_pos_embedder.enable_context_parallel(cp_group) + + if self.post_process: + self.final_layer = FinalLayer( + hidden_size=self.config.hidden_size, + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + out_channels=self.out_channels, + use_adaln_lora=self.use_adaln_lora, + adaln_lora_dim=self.adaln_lora_dim, + ) + + self.build_additional_timestamp_embedder() + # self.affline_norm = RMSNorm(self.config.hidden_size) + import transformer_engine as te + + self.affline_norm = te.pytorch.RMSNorm(self.config.hidden_size, eps=1e-6) + self.logvar = nn.Sequential( + FourierFeatures(num_channels=128, normalize=True), torch.nn.Linear(128, 1, bias=False) + ) + + def build_additional_timestamp_embedder(self): + if self.additional_timestamp_channels: + self.additional_timestamp_channels = dict(fps=256, h=256, w=256, org_h=256, org_w=256) + self.additional_timestamp_embedder = nn.ModuleDict() + for cond_name, cond_emb_channels in self.additional_timestamp_channels.items(): + print(f"Building additional timestamp embedder for {cond_name} with {cond_emb_channels} channels") + self.additional_timestamp_embedder[cond_name] = nn.Sequential( + SDXLTimesteps(cond_emb_channels), + SDXLTimestepEmbedding(cond_emb_channels, cond_emb_channels), + ) + + def prepare_additional_timestamp_embedder(self, **kwargs): + condition_concat = [] + for cond_name, embedder in self.additional_timestamp_embedder.items(): + condition_concat.append(embedder(kwargs[cond_name])) + embedding = torch.cat(condition_concat, dim=1) + if embedding.shape[1] < self.config.hidden_size: + embedding = nn.functional.pad(embedding, (0, self.config.hidden_size - embedding.shape[1])) + return embedding + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.concat_padding_mask: + padding_mask = padding_mask.squeeze(0) + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + if extra_pos_emb is not None: + extra_pos_emb = rearrange(extra_pos_emb, "B T H W D -> (T H W) B D") + return x_B_T_H_W_D, [self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb.detach()] + else: + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps) + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps.cuda()) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None + + def decoder_head( + self, + x_B_T_H_W_D: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + origin_shape: Tuple[int, int, int, int, int], # [B, C, T, H, W] + crossattn_mask: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + del crossattn_emb, crossattn_mask + B, C, T_before_patchify, H_before_patchify, W_before_patchify = origin_shape + # TODO: (qsh 2024-09-27) notation here is wrong, should be updated! + x_BT_HW_D = rearrange(x_B_T_H_W_D, "B T H W D -> (B T) (H W) D") + x_BT_HW_D = self.final_layer(x_BT_HW_D, emb_B_D, adaln_lora_B_3D=adaln_lora_B_3D) + # This is to ensure x_BT_HW_D has the correct shape because + # when we merge T, H, W into one dimension, x_BT_HW_D has shape (B * T * H * W, 1*1, D). + x_BT_HW_D = x_BT_HW_D.view( + B * T_before_patchify // self.patch_temporal, + H_before_patchify // self.patch_spatial * W_before_patchify // self.patch_spatial, + -1, + ) + x_B_D_T_H_W = rearrange( + x_BT_HW_D, + "(B T) (H W) (p1 p2 t C) -> B C (T t) (H p1) (W p2)", + p1=self.patch_spatial, + p2=self.patch_spatial, + H=H_before_patchify // self.patch_spatial, + W=W_before_patchify // self.patch_spatial, + t=self.patch_temporal, + B=B, + ) + return x_B_D_T_H_W + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + # Decoder forward + # Decoder embedding. + # print(f'x={x}') + # x = x.squeeze(0) + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + + rope_emb_L_1_1_D = None + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + B, T, H, W, D = x_B_T_H_W_D.shape + # print(f'x_T_H_W_B_D.shape={x_T_H_W_B_D.shape}') + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + # print(f'x_S_B_D.shape={x_S_B_D.shape}') + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # print(f'x_S_B_D={x_S_B_D}') + + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + packed_seq_params=packed_seq_params, + rotary_pos_emb=rope_emb_L_1_1_D[0], + ) + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + return x_B_T_H_W_D + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Sets input tensor to the model. + + See megatron.model.transformer.set_input_tensor() + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + assert len(input_tensor) == 1, "input_tensor should only be length 1 for gpt/bert" + self.decoder.set_input_tensor(input_tensor[0]) + + def sharded_state_dict( + self, prefix: str = "", sharded_offsets: tuple = (), metadata: Optional[Dict] = None + ) -> ShardedStateDict: + """Sharded state dict implementation for GPTModel backward-compatibility (removing extra state). + + Args: + prefix (str): Module name prefix. + sharded_offsets (tuple): PP related offsets, expected to be empty at this module level. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. + + Returns: + ShardedStateDict: sharded state dict for the GPTModel + """ + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) + + for param_name, param in self.t_embedder.named_parameters(): + weight_key = f"{prefix}t_embedder.{param_name}" + self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + # for cond_name, embedder in self.additional_timestamp_embedder.items(): + # for (param_name, param) in embedder.named_parameters(): + # weight_key = f'{prefix}additional_t_embedder_{cond_name}.{param_name}' + # self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + return sharded_state_dict + + def tie_embeddings_weights_state_dict( + self, + tensor, + sharded_state_dict: ShardedStateDict, + output_layer_weight_key: str, + first_stage_word_emb_key: str, + ) -> None: + """Ties the embedding and output weights in a given sharded state dict. + + Args: + sharded_state_dict (ShardedStateDict): state dict with the weight to tie + output_layer_weight_key (str): key of the output layer weight in the state dict. + This entry will be replaced with a tied version + first_stage_word_emb_key (str): this must be the same as the + ShardedTensor.key of the first stage word embeddings. + + Returns: None, acts in-place + """ + if self.pre_process and parallel_state.get_tensor_model_parallel_rank() == 0: + # Output layer is equivalent to the embedding already + return + + # Replace the default output layer with a one sharing the weights with the embedding + del sharded_state_dict[output_layer_weight_key] + last_stage_word_emb_replica_id = ( + 0, # copy of first stage embedding + parallel_state.get_tensor_model_parallel_rank() + + parallel_state.get_pipeline_model_parallel_rank() + * parallel_state.get_pipeline_model_parallel_world_size(), + parallel_state.get_data_parallel_rank(with_context_parallel=True), + ) + + sharded_state_dict[output_layer_weight_key] = make_sharded_tensor_for_checkpoint( + tensor=tensor, + key=first_stage_word_emb_key, + replica_id=last_stage_word_emb_replica_id, + allow_shape_mismatch=False, + ) + + +class DiTCrossAttentionExtendModel14B(VisionModule): + """DiT with CrossAttention model. + + Args: + config (TransformerConfig): transformer config + + transformer_decoder_layer_spec (ModuleSpec): transformer layer customization specs for decoder + + pre_process (bool): Include embedding layer (used with pipeline parallelism) + post_process (bool): Include an output layer (used with pipeline parallelism) + + fp16_lm_cross_entropy (bool, optional): Defaults to False + + parallel_output (bool): Do not gather the outputs, keep them split across tensor parallel ranks + + share_embeddings_and_output_weights (bool): When True, input embeddings and output logit weights are + shared. Defaults to False. + + position_embedding_type (string): Position embedding type. Options ['learned_absolute', 'rope']. + Defaults is 'learned_absolute'. + + rotary_percent (float): Percent of rotary dimension to use for rotary position embeddings. + Defaults to 1.0 (100%). Ignored unless position_embedding_type is 'rope'. + + seq_len_interpolation_factor (float): scale of linearly interpolating RoPE for longer sequences. + The value must be a float larger than 1.0. Defaults to None. + """ + + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + rotary_percent: float = 1.0, + seq_len_interpolation_factor: Optional[float] = None, + ): + super(DiTCrossAttentionExtendModel14B, self).__init__(config=config) + + self.config: TransformerConfig = config + from megatron.core.enums import ModelType + + self.model_type = ModelType.encoder_or_decoder + + self.transformer_decoder_layer_spec = DiTLayerWithAdaLNspec() + self.pre_process = pre_process + self.post_process = post_process + self.add_encoder = True + self.add_decoder = True + self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.parallel_output = parallel_output + self.position_embedding_type = position_embedding_type + self.share_embeddings_and_output_weights = False + self.additional_timestamp_channels = False + self.concat_padding_mask = True + self.pos_emb_cls = "rope3d" + self.use_adaln_lora = True + self.patch_spatial = 2 + self.patch_temporal = 1 + self.out_channels = 16 + self.adaln_lora_dim = 256 + + # Transformer decoder + self.decoder = TransformerBlock( + config=self.config, + spec=self.transformer_decoder_layer_spec, + pre_process=self.pre_process, + post_process=self.post_process, + post_layer_norm=False, + ) + + self.t_embedder = nn.Sequential( + SDXLTimesteps(self.config.hidden_size), + SDXLTimestepEmbedding( + self.config.hidden_size, self.config.hidden_size, use_adaln_lora=self.use_adaln_lora + ), + ) + + if self.pre_process: + self.in_channels = 17 + self.in_channels = self.in_channels + 1 if self.concat_padding_mask else self.in_channels + self.legacy_patch_emb = False + self.x_embedder = ( + PatchEmbed( + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + in_channels=self.in_channels, + out_channels=self.config.hidden_size, + bias=False, + keep_spatio=True, + legacy_patch_emb=self.legacy_patch_emb, + ) + .cuda() + .to(dtype=torch.bfloat16) + ) + + self.max_img_h = 240 + self.max_img_w = 240 + self.max_frames = 128 + self.min_fps = 1 + self.max_fps = 30 + self.pos_emb_learnable = False + self.pos_emb_interpolation = "crop" + self.rope_h_extrp_ratio = 2.0 + self.rope_w_extrp_ratio = 2.0 + self.rope_t_extrp_ratio = 2.0 + self.extra_per_block_abs_pos_emb = True + + self.pos_embedder = VideoRopePosition3DEmb( + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + max_fps=self.max_fps, + min_fps=self.min_fps, + is_learnable=self.pos_emb_learnable, + interpolation=self.pos_emb_interpolation, + head_dim=self.config.hidden_size // self.config.num_attention_heads, + h_extrp_ratio=self.rope_h_extrp_ratio, + w_extrp_ratio=self.rope_w_extrp_ratio, + t_extrp_ratio=self.rope_t_extrp_ratio, + ) + + if self.extra_per_block_abs_pos_emb: + self.extra_pos_embedder = SinCosPosEmbAxis( + h_extrapolation_ratio=2, + w_extrapolation_ratio=2, + t_extrapolation_ratio=2, + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + interpolation=self.pos_emb_interpolation, + ) + + if parallel_state.get_context_parallel_world_size() > 1: + cp_group = parallel_state.get_context_parallel_group() + self.pos_embedder.enable_context_parallel(cp_group) + self.extra_pos_embedder.enable_context_parallel(cp_group) + + if self.post_process: + self.final_layer = FinalLayer( + hidden_size=self.config.hidden_size, + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + out_channels=self.out_channels, + use_adaln_lora=self.use_adaln_lora, + adaln_lora_dim=self.adaln_lora_dim, + ) + + self.build_additional_timestamp_embedder() + # self.affline_norm = RMSNorm(self.config.hidden_size) + import transformer_engine as te + + self.affline_norm = te.pytorch.RMSNorm(self.config.hidden_size, eps=1e-6) + self.logvar = nn.Sequential( + FourierFeatures(num_channels=128, normalize=True), torch.nn.Linear(128, 1, bias=False) + ) + + def build_additional_timestamp_embedder(self): + if self.additional_timestamp_channels: + self.additional_timestamp_channels = dict(fps=256, h=256, w=256, org_h=256, org_w=256) + self.additional_timestamp_embedder = nn.ModuleDict() + for cond_name, cond_emb_channels in self.additional_timestamp_channels.items(): + print(f"Building additional timestamp embedder for {cond_name} with {cond_emb_channels} channels") + self.additional_timestamp_embedder[cond_name] = nn.Sequential( + SDXLTimesteps(cond_emb_channels), + SDXLTimestepEmbedding(cond_emb_channels, cond_emb_channels), + ) + + def prepare_additional_timestamp_embedder(self, **kwargs): + condition_concat = [] + for cond_name, embedder in self.additional_timestamp_embedder.items(): + condition_concat.append(embedder(kwargs[cond_name])) + embedding = torch.cat(condition_concat, dim=1) + if embedding.shape[1] < self.config.hidden_size: + embedding = nn.functional.pad(embedding, (0, self.config.hidden_size - embedding.shape[1])) + return embedding + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.concat_padding_mask: + padding_mask = padding_mask.squeeze(0) + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + if extra_pos_emb is not None: + extra_pos_emb = rearrange(extra_pos_emb, "B T H W D -> (T H W) B D") + return x_B_T_H_W_D, [self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb] + else: + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps) + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps.cuda()) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None + + def decoder_head( + self, + x_B_T_H_W_D: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + origin_shape: Tuple[int, int, int, int, int], # [B, C, T, H, W] + crossattn_mask: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + del crossattn_emb, crossattn_mask + B, C, T_before_patchify, H_before_patchify, W_before_patchify = origin_shape + # TODO: (qsh 2024-09-27) notation here is wrong, should be updated! + x_BT_HW_D = rearrange(x_B_T_H_W_D, "B T H W D -> (B T) (H W) D") + x_BT_HW_D = self.final_layer(x_BT_HW_D, emb_B_D, adaln_lora_B_3D=adaln_lora_B_3D) + # This is to ensure x_BT_HW_D has the correct shape because + # when we merge T, H, W into one dimension, x_BT_HW_D has shape (B * T * H * W, 1*1, D). + x_BT_HW_D = x_BT_HW_D.view( + B * T_before_patchify // self.patch_temporal, + H_before_patchify // self.patch_spatial * W_before_patchify // self.patch_spatial, + -1, + ) + x_B_D_T_H_W = rearrange( + x_BT_HW_D, + "(B T) (H W) (p1 p2 t C) -> B C (T t) (H p1) (W p2)", + p1=self.patch_spatial, + p2=self.patch_spatial, + H=H_before_patchify // self.patch_spatial, + W=W_before_patchify // self.patch_spatial, + t=self.patch_temporal, + B=B, + ) + return x_B_D_T_H_W + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + # Decoder forward + # Decoder embedding. + # print(f'x={x}') + # x = x.squeeze(0) + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + + input_list = [x, condition_video_input_mask] + x = torch.cat( + input_list, + dim=1, + ) + + rope_emb_L_1_1_D = None + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + B, T, H, W, D = x_B_T_H_W_D.shape + # print(f'x_T_H_W_B_D.shape={x_T_H_W_B_D.shape}') + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + # print(f'x_S_B_D.shape={x_S_B_D.shape}') + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # print(f'x_S_B_D={x_S_B_D}') + + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + packed_seq_params=packed_seq_params, + rotary_pos_emb=rope_emb_L_1_1_D[0], + ) + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + return x_B_T_H_W_D + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Sets input tensor to the model. + + See megatron.model.transformer.set_input_tensor() + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + assert len(input_tensor) == 1, "input_tensor should only be length 1 for gpt/bert" + self.decoder.set_input_tensor(input_tensor[0]) + + def sharded_state_dict( + self, prefix: str = "", sharded_offsets: tuple = (), metadata: Optional[Dict] = None + ) -> ShardedStateDict: + """Sharded state dict implementation for GPTModel backward-compatibility (removing extra state). + + Args: + prefix (str): Module name prefix. + sharded_offsets (tuple): PP related offsets, expected to be empty at this module level. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. + + Returns: + ShardedStateDict: sharded state dict for the GPTModel + """ + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) + + for param_name, param in self.t_embedder.named_parameters(): + weight_key = f"{prefix}t_embedder.{param_name}" + self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + # for cond_name, embedder in self.additional_timestamp_embedder.items(): + # for (param_name, param) in embedder.named_parameters(): + # weight_key = f'{prefix}additional_t_embedder_{cond_name}.{param_name}' + # self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + return sharded_state_dict + + def tie_embeddings_weights_state_dict( + self, + tensor, + sharded_state_dict: ShardedStateDict, + output_layer_weight_key: str, + first_stage_word_emb_key: str, + ) -> None: + """Ties the embedding and output weights in a given sharded state dict. + + Args: + sharded_state_dict (ShardedStateDict): state dict with the weight to tie + output_layer_weight_key (str): key of the output layer weight in the state dict. + This entry will be replaced with a tied version + first_stage_word_emb_key (str): this must be the same as the + ShardedTensor.key of the first stage word embeddings. + + Returns: None, acts in-place + """ + if self.pre_process and parallel_state.get_tensor_model_parallel_rank() == 0: + # Output layer is equivalent to the embedding already + return + + # Replace the default output layer with a one sharing the weights with the embedding + del sharded_state_dict[output_layer_weight_key] + last_stage_word_emb_replica_id = ( + 0, # copy of first stage embedding + parallel_state.get_tensor_model_parallel_rank() + + parallel_state.get_pipeline_model_parallel_rank() + * parallel_state.get_pipeline_model_parallel_world_size(), + parallel_state.get_data_parallel_rank(with_context_parallel=True), + ) + + sharded_state_dict[output_layer_weight_key] = make_sharded_tensor_for_checkpoint( + tensor=tensor, + key=first_stage_word_emb_key, + replica_id=last_stage_word_emb_replica_id, + allow_shape_mismatch=False, + ) + + +class DiTCrossAttentionActionExtendModel14B(DiTCrossAttentionExtendModel14B): + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + rotary_percent: float = 1.0, + seq_len_interpolation_factor: Optional[float] = None, + ): + # Initialize base V2W model. + super().__init__( + config, + pre_process, + post_process, + fp16_lm_cross_entropy, + parallel_output, + position_embedding_type, + rotary_percent, + seq_len_interpolation_factor, + ) + + self.action_mlp = ActionControlTorchMlp( + self.config.action_emb_dim, hidden_features=config.hidden_size, out_features=config.hidden_size + ) + self.action_mlp_3d = ActionControlTorchMlp( + self.config.action_emb_dim, + hidden_features=config.hidden_size, + out_features=config.hidden_size, + action_3d=True, + ) + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + action_control_condition: Tensor = None, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + **kwargs, + ) -> Tensor: + """Forward pass for the action-control V2W DiT model. + + Args: + x (Tensor): Tokenized video input of shape (B, C, T, H, W). + timestemps (Tensor): Timestep input tensor. + crossattn_emb (Tensor): Text embedding input for the cross-attention block. + action_control_condition (Tensor): Action control vector of shape (B, A), + where A is the action control dimensionality, e.g. A=7 for the bridge dataset. + + Returns: + Tensor: loss / noise prediction Tensor + """ + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + + input_list = [x, condition_video_input_mask] + x = torch.cat( + input_list, + dim=1, + ) + + rope_emb_L_1_1_D = None + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + B, T, H, W, D = x_B_T_H_W_D.shape + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + # Compute the action embedding MLP. + if action_control_condition is not None: + # Timestamp Embedding + action_embed = self.action_mlp(action_control_condition) + # Adaptive Layer Norm Embedding + action_embed_3d = self.action_mlp_3d(action_control_condition) + # Add action conditioning to affine embedding. + affline_emb_B_D = affline_emb_B_D + action_embed + if adaln_lora_B_3D is not None: + adaln_lora_B_3D = adaln_lora_B_3D + action_embed_3d + + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + packed_seq_params=packed_seq_params, + rotary_pos_emb=rope_emb_L_1_1_D[0], + ) + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + return x_B_T_H_W_D + + +# flake8: noqa: E741 diff --git a/nemo_vfm/diffusion/models/dit/dit_model_7b.py b/nemo_vfm/diffusion/models/dit/dit_model_7b.py new file mode 100755 index 00000000..a8814875 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit/dit_model_7b.py @@ -0,0 +1,2066 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +import warnings +from typing import Dict, List, Literal, Optional, Tuple + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from megatron.core import InferenceParams, parallel_state, tensor_parallel +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.utils import make_sharded_tensor_for_checkpoint +from nemo.collections.diffusion.models.dit.action_control.action_control_layers import ActionControlTorchMlp +from nemo.collections.diffusion.models.dit.cosmos_layer_spec import ( + get_dit_adaln_block_with_transformer_engine_spec as DiTLayerWithAdaLNspec, +) +from nemo.collections.diffusion.sampler.conditioner import DataType +from torch import Tensor +from torch.autograd import Function +from torch.distributed import ProcessGroup, all_gather, get_process_group_ranks, get_world_size +from torchvision import transforms + + +def gather_along_first_dim(tensor, process_group): + return AllGather.apply(tensor, process_group) + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class SDXLTimesteps(nn.Module): + def __init__(self, num_channels: int = 320): + super().__init__() + self.num_channels = num_channels + + def forward(self, timesteps): + in_dype = timesteps.dtype + half_dim = self.num_channels // 2 + exponent = -math.log(10000) * torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) + exponent = exponent / (half_dim - 0.0) + + emb = torch.exp(exponent) + emb = timesteps[:, None].float() * emb[None, :] + + sin_emb = torch.sin(emb) + cos_emb = torch.cos(emb) + emb = torch.cat([cos_emb, sin_emb], dim=-1) + + return emb.to(in_dype) + + +class SDXLTimestepEmbedding(nn.Module): + def __init__(self, in_features: int, out_features: int, use_adaln_lora: bool = False): + super().__init__() + self.linear_1 = nn.Linear(in_features, out_features, bias=not use_adaln_lora) + self.activation = nn.SiLU() + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.linear_2 = nn.Linear(out_features, 3 * out_features, bias=False) + else: + self.linear_2 = nn.Linear(out_features, out_features, bias=True) + + def forward(self, sample: torch.Tensor) -> torch.Tensor: + emb = self.linear_1(sample) + emb = self.activation(emb) + emb = self.linear_2(emb) + + if self.use_adaln_lora: + adaln_lora_B_3D = emb + emb_B_D = sample + else: + emb_B_D = emb + adaln_lora_B_3D = None + + return emb_B_D, adaln_lora_B_3D + + +class AllGather(Function): + @staticmethod + def forward(ctx, tensor, process_group): + world_size = dist.get_world_size(process_group) + ctx.world_size = world_size + ctx.rank = process_group.rank() + + gathered_tensors = [torch.zeros_like(tensor) for _ in range(world_size)] + dist.all_gather(gathered_tensors, tensor.contiguous(), process_group) + return torch.cat(gathered_tensors, dim=0) + + @staticmethod + def backward(ctx, grad_output): + world_size = ctx.world_size + rank = ctx.rank + + # Split the gradient tensor + grad_chunks = grad_output.chunk(world_size) + + # Select the gradient chunk for the current rank + grad_input = grad_chunks[rank] + return grad_input, None + + +class PatchEmbed(nn.Module): + # TODO: (qsh 2024-09-20) update docstring + """ + PatchEmbed is a module for embedding patches from an input tensor by applying either 3D or 2D convolutional layers, + depending on the . This module can process inputs with temporal (video) and spatial (image) dimensions, + making it suitable for video and image processing tasks. It supports dividing the input into patches and embedding each + patch into a vector of size `out_channels`. + + Parameters: + - spatial_patch_size (int): The size of each spatial patch. + - temporal_patch_size (int): The size of each temporal patch. + - in_channels (int): Number of input channels. Default: 3. + - out_channels (int): The dimension of the embedding vector for each patch. Default: 768. + - bias (bool): If True, adds a learnable bias to the output of the convolutional layers. Default: True. + - keep_spatio (bool): If True, the spatial dimensions are kept separate in the output tensor, otherwise, they are flattened. Default: False. + - legacy_patch_emb (bool): If True, applies 3D convolutional layers for video inputs, otherwise, use Linear! The legacy model is for backward compatibility. Default: True. + The output shape of the module depends on the `keep_spatio` flag. If `keep_spatio`=True, the output retains the spatial dimensions. + Otherwise, the spatial dimensions are flattened into a single dimension. + """ + + def __init__( + self, + spatial_patch_size, + temporal_patch_size, + in_channels=3, + out_channels=768, + bias=True, + keep_spatio=False, + legacy_patch_emb: bool = True, + ): + super().__init__() + self.spatial_patch_size = spatial_patch_size + self.temporal_patch_size = temporal_patch_size + assert keep_spatio, "Only support keep_spatio=True" + self.keep_spatio = keep_spatio + self.legacy_patch_emb = legacy_patch_emb + + if legacy_patch_emb: + self.proj = nn.Conv3d( + in_channels, + out_channels, + kernel_size=(temporal_patch_size, spatial_patch_size, spatial_patch_size), + stride=(temporal_patch_size, spatial_patch_size, spatial_patch_size), + bias=bias, + ) + self.out = Rearrange("b c t h w -> b t h w c") + else: + self.proj = nn.Sequential( + Rearrange( + "b c (t r) (h m) (w n) -> b t h w (c r m n)", + r=temporal_patch_size, + m=spatial_patch_size, + n=spatial_patch_size, + ), + nn.Linear( + in_channels * spatial_patch_size * spatial_patch_size * temporal_patch_size, + out_channels, + bias=bias, + ), + ) + self.out = nn.Identity() + + def forward(self, x): + """ + Forward pass of the PatchEmbed module. + + Parameters: + - x (torch.Tensor): The input tensor of shape (B, C, T, H, W) where + B is the batch size, + C is the number of channels, + T is the temporal dimension, + H is the height, and + W is the width of the input. + + Returns: + - torch.Tensor: The embedded patches as a tensor, with shape b t h w c. + """ + assert x.dim() == 5 + _, _, T, H, W = x.shape + assert H % self.spatial_patch_size == 0 and W % self.spatial_patch_size == 0 + assert T % self.temporal_patch_size == 0 + x = self.proj(x) + return self.out(x) + + +class FourierFeatures(nn.Module): + """ + Implements a layer that generates Fourier features from input tensors, based on randomly sampled + frequencies and phases. This can help in learning high-frequency functions in low-dimensional problems. + + [B] -> [B, D] + + Parameters: + num_channels (int): The number of Fourier features to generate. + bandwidth (float, optional): The scaling factor for the frequency of the Fourier features. Defaults to 1. + normalize (bool, optional): If set to True, the outputs are scaled by sqrt(2), usually to normalize + the variance of the features. Defaults to False. + + Example: + >>> layer = FourierFeatures(num_channels=256, bandwidth=0.5, normalize=True) + >>> x = torch.randn(10, 256) # Example input tensor + >>> output = layer(x) + >>> print(output.shape) # Expected shape: (10, 256) + """ + + def __init__(self, num_channels, bandwidth=1, normalize=False): + super().__init__() + self.register_buffer("freqs", 2 * np.pi * bandwidth * torch.randn(num_channels), persistent=True) + self.register_buffer("phases", 2 * np.pi * torch.rand(num_channels), persistent=True) + self.gain = np.sqrt(2) if normalize else 1 + + def forward(self, x, gain: float = 1.0): + """ + Apply the Fourier feature transformation to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + gain (float, optional): An additional gain factor applied during the forward pass. Defaults to 1. + + Returns: + torch.Tensor: The transformed tensor, with Fourier features applied. + """ + in_dtype = x.dtype + x = x.to(torch.float32).ger(self.freqs.to(torch.float32)).add(self.phases.to(torch.float32)) + x = x.cos().mul(self.gain * gain).to(in_dtype) + return x + + +class FinalLayer(nn.Module): + """ + The final layer of video DiT. + """ + + def __init__( + self, + hidden_size, + spatial_patch_size, + temporal_patch_size, + out_channels, + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + ): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear( + hidden_size, spatial_patch_size * spatial_patch_size * temporal_patch_size * out_channels, bias=False + ) + self.hidden_size = hidden_size + self.n_adaln_chunks = 2 + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, adaln_lora_dim, bias=False), + nn.Linear(adaln_lora_dim, self.n_adaln_chunks * hidden_size, bias=False), + ) + else: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, self.n_adaln_chunks * hidden_size, bias=False) + ) + + self.sequence_parallel = getattr(parallel_state, "sequence_parallel", False) + + def forward( + self, + x_BT_HW_D, + emb_B_D, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ): + if self.use_adaln_lora: + assert adaln_lora_B_3D is not None + shift_B_D, scale_B_D = (self.adaLN_modulation(emb_B_D) + adaln_lora_B_3D[:, : 2 * self.hidden_size]).chunk( + 2, dim=1 + ) + else: + shift_B_D, scale_B_D = self.adaLN_modulation(emb_B_D).chunk(2, dim=1) + + B = emb_B_D.shape[0] + T = x_BT_HW_D.shape[0] // B + shift_BT_D, scale_BT_D = repeat(shift_B_D, "b d -> (b t) d", t=T), repeat(scale_B_D, "b d -> (b t) d", t=T) + x_BT_HW_D = modulate(self.norm_final(x_BT_HW_D), shift_BT_D, scale_BT_D) + if self.sequence_parallel: + x_T_B_HW_D = rearrange(x_BT_HW_D, "(b t) hw d -> t b hw d", b=B, t=T) + x_T_B_HW_D = gather_along_first_dim(x_T_B_HW_D, parallel_state.get_tensor_model_parallel_group()) + x_BT_HW_D = rearrange(x_T_B_HW_D, "t b hw d -> (b t) hw d", b=B) + + x_BT_HW_D = self.linear(x_BT_HW_D) + return x_BT_HW_D + + +class RMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()).type_as(x) + return output * self.weight + + +def split_inputs_cp(x: Tensor, seq_dim: int, cp_group: ProcessGroup) -> Tensor: + """ + Split input tensor along the sequence dimension for checkpoint parallelism. + + This function divides the input tensor into equal parts along the specified + sequence dimension, based on the number of ranks in the checkpoint parallelism group. + It then selects the part corresponding to the current rank. + + Args: + x: Input tensor to be split. + seq_dim: The dimension along which to split the input (sequence dimension). + cp_group: The process group for checkpoint parallelism. + + Returns: + A slice of the input tensor corresponding to the current rank. + + Raises: + AssertionError: If the sequence dimension is not divisible by the number of ranks. + """ + cp_ranks = get_process_group_ranks(cp_group) + cp_size = len(cp_ranks) + + assert x.shape[seq_dim] % cp_size == 0, f"{x.shape[seq_dim]} cannot divide cp_size {cp_size}" + x = x.view(*x.shape[:seq_dim], cp_size, x.shape[seq_dim] // cp_size, *x.shape[(seq_dim + 1) :]) + seq_idx = torch.tensor([cp_group.rank()], device=x.device) + x = x.index_select(seq_dim, seq_idx) + # Note that the new sequence length is the original sequence length / cp_size + x = x.view(*x.shape[:seq_dim], -1, *x.shape[(seq_dim + 2) :]) + return x + + +def cat_outputs_cp(x: Tensor, seq_dim: int, cp_group: ProcessGroup) -> Tensor: + """ + Concatenates tensors from multiple processes along a specified dimension. + + This function gathers tensors from all processes in the given process group + and concatenates them along the specified dimension. + + Args: + x (Tensor): The input tensor to be gathered and concatenated. + seq_dim (int): The dimension along which to concatenate the gathered tensors. + cp_group (ProcessGroup): The process group containing all the processes involved in the gathering. + + Returns: + Tensor: A tensor resulting from the concatenation of tensors from all processes. + + Raises: + RuntimeError: If the gathering of tensors fails. + """ + # Number of processes in the group + world_size = get_world_size(cp_group) + + # List to hold tensors from each rank + gathered_tensors = [torch.zeros_like(x) for _ in range(world_size)] + + # Attempt to gather tensors from all ranks + try: + all_gather(gathered_tensors, x, group=cp_group) + except RuntimeError as e: + raise RuntimeError(f"Gathering failed: {e}") + + # Concatenate tensors along the specified dimension + return torch.cat(gathered_tensors, dim=seq_dim) + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +def normalize(x: torch.Tensor, dim: Optional[List[int]] = None, eps: float = 0) -> torch.Tensor: + """ + Normalizes the input tensor along specified dimensions such that the average square norm of elements is adjusted. + + Args: + x (torch.Tensor): The input tensor to normalize. + dim (list, optional): The dimensions over which to normalize. If None, normalizes over all dimensions except the first. + eps (float, optional): A small constant to ensure numerical stability during division. + + Returns: + torch.Tensor: The normalized tensor. + """ + if dim is None: + dim = list(range(1, x.ndim)) + norm = torch.linalg.vector_norm(x, dim=dim, keepdim=True, dtype=torch.float32) + norm = torch.add(eps, norm, alpha=np.sqrt(norm.numel() / x.numel())) + return x / norm.to(x.dtype) + + +class VideoPositionEmb(nn.Module): + def __init__(self): + super().__init__() + self.cp_group = None + + def enable_context_parallel(self, cp_group: ProcessGroup): + self.cp_group = cp_group + + def disable_context_parallel(self): + self.cp_group = None + + def forward(self, x_B_T_H_W_C: torch.Tensor, fps=Optional[torch.Tensor]) -> torch.Tensor: + """ + With CP, the function assume that the input tensor is already split. It delegates the embedding generation to generate_embeddings function. + """ + B_T_H_W_C = x_B_T_H_W_C.shape + if self.cp_group is not None: + cp_ranks = get_process_group_ranks(self.cp_group) + cp_size = len(cp_ranks) + B, T, H, W, C = B_T_H_W_C + B_T_H_W_C = (B, T * cp_size, H, W, C) + embeddings = self.generate_embeddings(B_T_H_W_C, fps=fps) + + if self.cp_group is not None: + if isinstance(self, VideoRopePosition3DEmb): + seq_dim = 0 + else: + seq_dim = 1 + embeddings = split_inputs_cp(x=embeddings, seq_dim=seq_dim, cp_group=self.cp_group) + return embeddings + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]): + raise NotImplementedError + + +class VideoRopePosition3DEmb(VideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + head_dim: int, + len_h: int, + len_w: int, + len_t: int, + base_fps: int = 24, + h_extrp_ratio: float = 1.0, + w_extrp_ratio: float = 1.0, + t_extrp_ratio: float = 1.0, + **kwargs, # used for compatibility with other positional embeddings; unused in this class + ): + del kwargs + super().__init__() + self.register_buffer("seq", torch.arange(max(len_h, len_w, len_t), dtype=torch.float)) + self.base_fps = base_fps + self.max_h = len_h + self.max_w = len_w + + dim = head_dim + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + self.register_buffer( + "dim_spatial_range", + torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().cuda() / dim_h, + persistent=False, + ) + self.register_buffer( + "dim_temporal_range", + torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().cuda() / dim_t, + persistent=False, + ) + + self.h_ntk_factor = h_extrp_ratio ** (dim_h / (dim_h - 2)) + self.w_ntk_factor = w_extrp_ratio ** (dim_w / (dim_w - 2)) + self.t_ntk_factor = t_extrp_ratio ** (dim_t / (dim_t - 2)) + + def generate_embeddings( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. Defaults to None. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. Defaults to None. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. Defaults to None. + + Returns: + Not specified in the original code snippet. + """ + h_ntk_factor = h_ntk_factor if h_ntk_factor is not None else self.h_ntk_factor + w_ntk_factor = w_ntk_factor if w_ntk_factor is not None else self.w_ntk_factor + t_ntk_factor = t_ntk_factor if t_ntk_factor is not None else self.t_ntk_factor + + h_theta = 10000.0 * h_ntk_factor + w_theta = 10000.0 * w_ntk_factor + t_theta = 10000.0 * t_ntk_factor + + h_spatial_freqs = 1.0 / (h_theta**self.dim_spatial_range) + w_spatial_freqs = 1.0 / (w_theta**self.dim_spatial_range) + temporal_freqs = 1.0 / (t_theta**self.dim_temporal_range) + + B, T, H, W, _ = B_T_H_W_C + assert B == 1 or T == 1, ( + "Batch size should be 1 or T should be 1. Image batch should have T=1, while video batch should have B=1." + ) + assert H <= self.max_h and W <= self.max_w, ( + f"Input dimensions (H={H}, W={W}) exceed the maximum dimensions (max_h={self.max_h}, max_w={self.max_w}) configured for positional embedding. Please adjust the input size or increase the maximum dimensions in the model configuration." + ) + self.seq = self.seq.cuda() + half_emb_h = torch.outer(self.seq[:H], h_spatial_freqs) + half_emb_w = torch.outer(self.seq[:W], w_spatial_freqs) + + # apply sequence scaling in temporal dimension + if fps is None: # image case + assert T == 1, "T should be 1 for image batch." + half_emb_t = torch.outer(self.seq[:T], temporal_freqs) + else: + half_emb_t = torch.outer(self.seq[:T] / fps[:1] * self.base_fps, temporal_freqs) + + em_T_H_W_D = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + + return rearrange(em_T_H_W_D, "t h w d -> (t h w) 1 1 d").float() + + +class SinCosPosEmbAxis(VideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + interpolation: str, + model_channels: int, + len_h: int, + len_w: int, + len_t: int, + is_learnable: bool = True, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + **kwargs, + ): + # TODO: (qsh 2024-11-08) add more interpolation methods and args for extrapolation fine-tuning + """ + Args: + interpolation (str): we curretly only support "crop", ideally when we need extrapolation capacity, we should adjust frequency or other more advanced methods. they are not implemented yet. + """ + del kwargs # unused + super().__init__() + self.interpolation = interpolation + self.is_learnable = is_learnable + assert self.interpolation in ["crop"], f"Unknown interpolation method {self.interpolation}" + + self.pos_emb_h = nn.Parameter(torch.zeros(len_h, model_channels)) + self.pos_emb_w = nn.Parameter(torch.zeros(len_w, model_channels)) + self.pos_emb_t = nn.Parameter(torch.zeros(len_t, model_channels)) + + trunc_normal_(self.pos_emb_h, std=0.02) + trunc_normal_(self.pos_emb_w, std=0.02) + trunc_normal_(self.pos_emb_t, std=0.02) + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]) -> torch.Tensor: + B, T, H, W, C = B_T_H_W_C + if self.interpolation == "crop": + emb_h_H = self.pos_emb_h[:H] + emb_w_W = self.pos_emb_w[:W] + emb_t_T = self.pos_emb_t[:T] + emb = ( + repeat(emb_t_T, "t d-> b t h w d", b=B, h=H, w=W) + + repeat(emb_h_H, "h d-> b t h w d", b=B, t=T, w=W) + + repeat(emb_w_W, "w d-> b t h w d", b=B, t=T, h=H) + ) + assert list(emb.shape)[:4] == [B, T, H, W], f"bad shape: {list(emb.shape)[:4]} != {B, T, H, W}" + else: + raise ValueError(f"Unknown interpolation method {self.interpolation}") + + return normalize(emb, dim=-1, eps=1e-6) + + +class DiTCrossAttentionModel7B(VisionModule): + """DiT with CrossAttention model. + + Args: + config (TransformerConfig): transformer config + + transformer_decoder_layer_spec (ModuleSpec): transformer layer customization specs for decoder + + pre_process (bool): Include embedding layer (used with pipeline parallelism) + post_process (bool): Include an output layer (used with pipeline parallelism) + + fp16_lm_cross_entropy (bool, optional): Defaults to False + + parallel_output (bool): Do not gather the outputs, keep them split across tensor parallel ranks + + share_embeddings_and_output_weights (bool): When True, input embeddings and output logit weights are + shared. Defaults to False. + + position_embedding_type (string): Position embedding type. Options ['learned_absolute', 'rope']. + Defaults is 'learned_absolute'. + + rotary_percent (float): Percent of rotary dimension to use for rotary position embeddings. + Defaults to 1.0 (100%). Ignored unless position_embedding_type is 'rope'. + + seq_len_interpolation_factor (float): scale of linearly interpolating RoPE for longer sequences. + The value must be a float larger than 1.0. Defaults to None. + """ + + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + rotary_percent: float = 1.0, + seq_len_interpolation_factor: Optional[float] = None, + ): + super(DiTCrossAttentionModel7B, self).__init__(config=config) + + self.config: TransformerConfig = config + from megatron.core.enums import ModelType + + self.model_type = ModelType.encoder_or_decoder + + self.transformer_decoder_layer_spec = DiTLayerWithAdaLNspec() + self.pre_process = pre_process + self.post_process = post_process + self.add_encoder = True + self.add_decoder = True + self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.parallel_output = parallel_output + self.position_embedding_type = position_embedding_type + self.share_embeddings_and_output_weights = False + self.additional_timestamp_channels = False + self.concat_padding_mask = True + self.pos_emb_cls = "rope3d" + self.use_adaln_lora = True + self.patch_spatial = 2 + self.patch_temporal = 1 + self.out_channels = 16 + self.adaln_lora_dim = 256 + + # Transformer decoder + self.decoder = TransformerBlock( + config=self.config, + spec=self.transformer_decoder_layer_spec, + pre_process=self.pre_process, + post_process=self.post_process, + post_layer_norm=False, + ) + + self.t_embedder = nn.Sequential( + SDXLTimesteps(self.config.hidden_size), + SDXLTimestepEmbedding( + self.config.hidden_size, self.config.hidden_size, use_adaln_lora=self.use_adaln_lora + ), + ) + + if self.pre_process: + self.in_channels = 16 + self.in_channels = self.in_channels + 1 if self.concat_padding_mask else self.in_channels + self.legacy_patch_emb = False + self.x_embedder = ( + PatchEmbed( + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + in_channels=self.in_channels, + out_channels=self.config.hidden_size, + bias=False, + keep_spatio=True, + legacy_patch_emb=self.legacy_patch_emb, + ) + .cuda() + .to(dtype=torch.bfloat16) + ) + + self.max_img_h = 240 + self.max_img_w = 240 + self.max_frames = 128 + self.min_fps = 1 + self.max_fps = 30 + self.pos_emb_learnable = False + self.pos_emb_interpolation = "crop" + self.rope_h_extrp_ratio = 1.0 + self.rope_w_extrp_ratio = 1.0 + self.rope_t_extrp_ratio = 2.0 + self.extra_per_block_abs_pos_emb = True + + self.pos_embedder = VideoRopePosition3DEmb( + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + max_fps=self.max_fps, + min_fps=self.min_fps, + is_learnable=self.pos_emb_learnable, + interpolation=self.pos_emb_interpolation, + head_dim=self.config.hidden_size // self.config.num_attention_heads, + h_extrp_ratio=self.rope_h_extrp_ratio, + w_extrp_ratio=self.rope_w_extrp_ratio, + t_extrp_ratio=self.rope_t_extrp_ratio, + ) + + if self.extra_per_block_abs_pos_emb: + self.extra_pos_embedder = SinCosPosEmbAxis( + h_extrapolation_ratio=1, + w_extrapolation_ratio=1, + t_extrapolation_ratio=1, + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + interpolation=self.pos_emb_interpolation, + ) + + if parallel_state.get_context_parallel_world_size() > 1: + cp_group = parallel_state.get_context_parallel_group() + self.pos_embedder.enable_context_parallel(cp_group) + self.extra_pos_embedder.enable_context_parallel(cp_group) + + if self.post_process: + self.final_layer = FinalLayer( + hidden_size=self.config.hidden_size, + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + out_channels=self.out_channels, + use_adaln_lora=self.use_adaln_lora, + adaln_lora_dim=self.adaln_lora_dim, + ) + + self.build_additional_timestamp_embedder() + # self.affline_norm = RMSNorm(self.config.hidden_size) + import transformer_engine as te + + self.affline_norm = te.pytorch.RMSNorm(self.config.hidden_size, eps=1e-6) + self.logvar = nn.Sequential( + FourierFeatures(num_channels=128, normalize=True), torch.nn.Linear(128, 1, bias=False) + ) + + def build_additional_timestamp_embedder(self): + if self.additional_timestamp_channels: + self.additional_timestamp_channels = dict(fps=256, h=256, w=256, org_h=256, org_w=256) + self.additional_timestamp_embedder = nn.ModuleDict() + for cond_name, cond_emb_channels in self.additional_timestamp_channels.items(): + print(f"Building additional timestamp embedder for {cond_name} with {cond_emb_channels} channels") + self.additional_timestamp_embedder[cond_name] = nn.Sequential( + SDXLTimesteps(cond_emb_channels), + SDXLTimestepEmbedding(cond_emb_channels, cond_emb_channels), + ) + + def prepare_additional_timestamp_embedder(self, **kwargs): + condition_concat = [] + for cond_name, embedder in self.additional_timestamp_embedder.items(): + condition_concat.append(embedder(kwargs[cond_name])) + embedding = torch.cat(condition_concat, dim=1) + if embedding.shape[1] < self.config.hidden_size: + embedding = nn.functional.pad(embedding, (0, self.config.hidden_size - embedding.shape[1])) + return embedding + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.concat_padding_mask: + padding_mask = padding_mask.squeeze(0) + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + if extra_pos_emb is not None: + extra_pos_emb = rearrange(extra_pos_emb, "B T H W D -> (T H W) B D") + return x_B_T_H_W_D, [self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb] + else: + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps) + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps.cuda()) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None + + def decoder_head( + self, + x_B_T_H_W_D: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + origin_shape: Tuple[int, int, int, int, int], # [B, C, T, H, W] + crossattn_mask: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + del crossattn_emb, crossattn_mask + B, C, T_before_patchify, H_before_patchify, W_before_patchify = origin_shape + # TODO: (qsh 2024-09-27) notation here is wrong, should be updated! + x_BT_HW_D = rearrange(x_B_T_H_W_D, "B T H W D -> (B T) (H W) D") + x_BT_HW_D = self.final_layer(x_BT_HW_D, emb_B_D, adaln_lora_B_3D=adaln_lora_B_3D) + # This is to ensure x_BT_HW_D has the correct shape because + # when we merge T, H, W into one dimension, x_BT_HW_D has shape (B * T * H * W, 1*1, D). + x_BT_HW_D = x_BT_HW_D.view( + B * T_before_patchify // self.patch_temporal, + H_before_patchify // self.patch_spatial * W_before_patchify // self.patch_spatial, + -1, + ) + x_B_D_T_H_W = rearrange( + x_BT_HW_D, + "(B T) (H W) (p1 p2 t C) -> B C (T t) (H p1) (W p2)", + p1=self.patch_spatial, + p2=self.patch_spatial, + H=H_before_patchify // self.patch_spatial, + W=W_before_patchify // self.patch_spatial, + t=self.patch_temporal, + B=B, + ) + return x_B_D_T_H_W + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + # Decoder forward + # Decoder embedding. + # print(f'x={x}') + # x = x.squeeze(0) + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + + rope_emb_L_1_1_D = None + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + B, T, H, W, D = x_B_T_H_W_D.shape + # print(f'x_T_H_W_B_D.shape={x_T_H_W_B_D.shape}') + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + # print(f'x_S_B_D.shape={x_S_B_D.shape}') + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # print(f'x_S_B_D={x_S_B_D}') + + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + packed_seq_params=packed_seq_params, + rotary_pos_emb=rope_emb_L_1_1_D[0], + ) + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + return x_B_T_H_W_D + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Sets input tensor to the model. + + See megatron.model.transformer.set_input_tensor() + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + assert len(input_tensor) == 1, "input_tensor should only be length 1 for gpt/bert" + self.decoder.set_input_tensor(input_tensor[0]) + + def sharded_state_dict( + self, prefix: str = "", sharded_offsets: tuple = (), metadata: Optional[Dict] = None + ) -> ShardedStateDict: + """Sharded state dict implementation for GPTModel backward-compatibility (removing extra state). + + Args: + prefix (str): Module name prefix. + sharded_offsets (tuple): PP related offsets, expected to be empty at this module level. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. + + Returns: + ShardedStateDict: sharded state dict for the GPTModel + """ + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) + + for param_name, param in self.t_embedder.named_parameters(): + weight_key = f"{prefix}t_embedder.{param_name}" + self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + # for cond_name, embedder in self.additional_timestamp_embedder.items(): + # for (param_name, param) in embedder.named_parameters(): + # weight_key = f'{prefix}additional_t_embedder_{cond_name}.{param_name}' + # self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + return sharded_state_dict + + def tie_embeddings_weights_state_dict( + self, + tensor, + sharded_state_dict: ShardedStateDict, + output_layer_weight_key: str, + first_stage_word_emb_key: str, + ) -> None: + """Ties the embedding and output weights in a given sharded state dict. + + Args: + sharded_state_dict (ShardedStateDict): state dict with the weight to tie + output_layer_weight_key (str): key of the output layer weight in the state dict. + This entry will be replaced with a tied version + first_stage_word_emb_key (str): this must be the same as the + ShardedTensor.key of the first stage word embeddings. + + Returns: None, acts in-place + """ + if self.pre_process and parallel_state.get_tensor_model_parallel_rank() == 0: + # Output layer is equivalent to the embedding already + return + + # Replace the default output layer with a one sharing the weights with the embedding + del sharded_state_dict[output_layer_weight_key] + last_stage_word_emb_replica_id = ( + 0, # copy of first stage embedding + parallel_state.get_tensor_model_parallel_rank() + + parallel_state.get_pipeline_model_parallel_rank() + * parallel_state.get_pipeline_model_parallel_world_size(), + parallel_state.get_data_parallel_rank(with_context_parallel=True), + ) + + sharded_state_dict[output_layer_weight_key] = make_sharded_tensor_for_checkpoint( + tensor=tensor, + key=first_stage_word_emb_key, + replica_id=last_stage_word_emb_replica_id, + allow_shape_mismatch=False, + ) + + +class DiTCrossAttentionExtendModel7B(VisionModule): + """DiT with CrossAttention model. + + Args: + config (TransformerConfig): transformer config + + transformer_decoder_layer_spec (ModuleSpec): transformer layer customization specs for decoder + + pre_process (bool): Include embedding layer (used with pipeline parallelism) + post_process (bool): Include an output layer (used with pipeline parallelism) + + fp16_lm_cross_entropy (bool, optional): Defaults to False + + parallel_output (bool): Do not gather the outputs, keep them split across tensor parallel ranks + + share_embeddings_and_output_weights (bool): When True, input embeddings and output logit weights are + shared. Defaults to False. + + position_embedding_type (string): Position embedding type. Options ['learned_absolute', 'rope']. + Defaults is 'learned_absolute'. + + rotary_percent (float): Percent of rotary dimension to use for rotary position embeddings. + Defaults to 1.0 (100%). Ignored unless position_embedding_type is 'rope'. + + seq_len_interpolation_factor (float): scale of linearly interpolating RoPE for longer sequences. + The value must be a float larger than 1.0. Defaults to None. + """ + + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + rotary_percent: float = 1.0, + seq_len_interpolation_factor: Optional[float] = None, + ): + super().__init__(config=config) + + self.config: TransformerConfig = config + from megatron.core.enums import ModelType + + self.model_type = ModelType.encoder_or_decoder + + self.transformer_decoder_layer_spec = DiTLayerWithAdaLNspec() + self.pre_process = pre_process + self.post_process = post_process + self.add_encoder = True + self.add_decoder = True + self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.parallel_output = parallel_output + self.position_embedding_type = position_embedding_type + self.share_embeddings_and_output_weights = False + self.additional_timestamp_channels = False + self.concat_padding_mask = True + self.pos_emb_cls = "rope3d" + self.use_adaln_lora = True + self.patch_spatial = 2 + self.patch_temporal = 1 + self.out_channels = 16 + self.adaln_lora_dim = 256 + + # Transformer decoder + self.decoder = TransformerBlock( + config=self.config, + spec=self.transformer_decoder_layer_spec, + pre_process=self.pre_process, + post_process=self.post_process, + post_layer_norm=False, + ) + + self.t_embedder = nn.Sequential( + SDXLTimesteps(self.config.hidden_size), + SDXLTimestepEmbedding( + self.config.hidden_size, self.config.hidden_size, use_adaln_lora=self.use_adaln_lora + ), + ) + + if self.pre_process: + self.in_channels = 17 + self.in_channels = self.in_channels + 1 if self.concat_padding_mask else self.in_channels + self.legacy_patch_emb = False + self.x_embedder = ( + PatchEmbed( + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + in_channels=self.in_channels, + out_channels=self.config.hidden_size, + bias=False, + keep_spatio=True, + legacy_patch_emb=self.legacy_patch_emb, + ) + .cuda() + .to(dtype=torch.bfloat16) + ) + + self.max_img_h = 240 + self.max_img_w = 240 + self.max_frames = 128 + self.min_fps = 1 + self.max_fps = 30 + self.pos_emb_learnable = False + self.pos_emb_interpolation = "crop" + self.rope_h_extrp_ratio = 1.0 + self.rope_w_extrp_ratio = 1.0 + self.rope_t_extrp_ratio = 2.0 + self.extra_per_block_abs_pos_emb = True + + self.pos_embedder = VideoRopePosition3DEmb( + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + max_fps=self.max_fps, + min_fps=self.min_fps, + is_learnable=self.pos_emb_learnable, + interpolation=self.pos_emb_interpolation, + head_dim=self.config.hidden_size // self.config.num_attention_heads, + h_extrp_ratio=self.rope_h_extrp_ratio, + w_extrp_ratio=self.rope_w_extrp_ratio, + t_extrp_ratio=self.rope_t_extrp_ratio, + ) + + if self.extra_per_block_abs_pos_emb: + self.extra_pos_embedder = SinCosPosEmbAxis( + h_extrapolation_ratio=1, + w_extrapolation_ratio=1, + t_extrapolation_ratio=1, + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + interpolation=self.pos_emb_interpolation, + ) + + if parallel_state.get_context_parallel_world_size() > 1: + cp_group = parallel_state.get_context_parallel_group() + self.pos_embedder.enable_context_parallel(cp_group) + self.extra_pos_embedder.enable_context_parallel(cp_group) + + if self.post_process: + self.final_layer = FinalLayer( + hidden_size=self.config.hidden_size, + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + out_channels=self.out_channels, + use_adaln_lora=self.use_adaln_lora, + adaln_lora_dim=self.adaln_lora_dim, + ) + + self.build_additional_timestamp_embedder() + # self.affline_norm = RMSNorm(self.config.hidden_size) + import transformer_engine as te + + self.affline_norm = te.pytorch.RMSNorm(self.config.hidden_size, eps=1e-6) + self.logvar = nn.Sequential( + FourierFeatures(num_channels=128, normalize=True), torch.nn.Linear(128, 1, bias=False) + ) + + def build_additional_timestamp_embedder(self): + if self.additional_timestamp_channels: + self.additional_timestamp_channels = dict(fps=256, h=256, w=256, org_h=256, org_w=256) + self.additional_timestamp_embedder = nn.ModuleDict() + for cond_name, cond_emb_channels in self.additional_timestamp_channels.items(): + print(f"Building additional timestamp embedder for {cond_name} with {cond_emb_channels} channels") + self.additional_timestamp_embedder[cond_name] = nn.Sequential( + SDXLTimesteps(cond_emb_channels), + SDXLTimestepEmbedding(cond_emb_channels, cond_emb_channels), + ) + + def prepare_additional_timestamp_embedder(self, **kwargs): + condition_concat = [] + for cond_name, embedder in self.additional_timestamp_embedder.items(): + condition_concat.append(embedder(kwargs[cond_name])) + embedding = torch.cat(condition_concat, dim=1) + if embedding.shape[1] < self.config.hidden_size: + embedding = nn.functional.pad(embedding, (0, self.config.hidden_size - embedding.shape[1])) + return embedding + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.concat_padding_mask: + padding_mask = padding_mask.squeeze(0) + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + if extra_pos_emb is not None: + extra_pos_emb = rearrange(extra_pos_emb, "B T H W D -> (T H W) B D") + return x_B_T_H_W_D, [self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb] + else: + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps) + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps.cuda()) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None + + def decoder_head( + self, + x_B_T_H_W_D: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + origin_shape: Tuple[int, int, int, int, int], # [B, C, T, H, W] + crossattn_mask: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + del crossattn_emb, crossattn_mask + B, C, T_before_patchify, H_before_patchify, W_before_patchify = origin_shape + # TODO: (qsh 2024-09-27) notation here is wrong, should be updated! + x_BT_HW_D = rearrange(x_B_T_H_W_D, "B T H W D -> (B T) (H W) D") + x_BT_HW_D = self.final_layer(x_BT_HW_D, emb_B_D, adaln_lora_B_3D=adaln_lora_B_3D) + # This is to ensure x_BT_HW_D has the correct shape because + # when we merge T, H, W into one dimension, x_BT_HW_D has shape (B * T * H * W, 1*1, D). + x_BT_HW_D = x_BT_HW_D.view( + B * T_before_patchify // self.patch_temporal, + H_before_patchify // self.patch_spatial * W_before_patchify // self.patch_spatial, + -1, + ) + x_B_D_T_H_W = rearrange( + x_BT_HW_D, + "(B T) (H W) (p1 p2 t C) -> B C (T t) (H p1) (W p2)", + p1=self.patch_spatial, + p2=self.patch_spatial, + H=H_before_patchify // self.patch_spatial, + W=W_before_patchify // self.patch_spatial, + t=self.patch_temporal, + B=B, + ) + return x_B_D_T_H_W + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + condition_video_pose: Optional[torch.Tensor] = None, + x_ctrl: Optional[Dict[str, torch.Tensor]] = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + # Decoder forward + # Decoder embedding. + # print(f'x={x}') + # x = x.squeeze(0) + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + + input_list = [x, condition_video_input_mask] + if condition_video_pose is not None: + if condition_video_pose.shape[2] > T: + condition_video_pose = condition_video_pose[:, :, :T, :, :].contiguous() + input_list.append(condition_video_pose) + x = torch.cat( + input_list, + dim=1, + ) + + rope_emb_L_1_1_D = None + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + B, T, H, W, D = x_B_T_H_W_D.shape + # print(f'x_T_H_W_B_D.shape={x_T_H_W_B_D.shape}') + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + # print(f'x_S_B_D.shape={x_S_B_D.shape}') + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # print(f'x_S_B_D={x_S_B_D}') + + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + for idx, block in enumerate(self.decoder.layers): + name = f"block{idx}" + x_S_B_D, _ = block( + x_S_B_D, + affline_emb_B_D, + crossattn_emb, + None, + rotary_pos_emb=rope_emb_L_1_1_D[0], + packed_seq_params=packed_seq_params, + ) + if x_ctrl is not None and name in x_ctrl: + # x_ctrl_S_B_D = rearrange(x_ctrl[name], "A B C D E -> (A B C) D E") + x_ctrl_S_B_D = x_ctrl[name] + x_S_B_D = x_S_B_D + x_ctrl_S_B_D + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + return x_B_T_H_W_D + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Sets input tensor to the model. + + See megatron.model.transformer.set_input_tensor() + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + assert len(input_tensor) == 1, "input_tensor should only be length 1 for gpt/bert" + self.decoder.set_input_tensor(input_tensor[0]) + + def sharded_state_dict( + self, prefix: str = "", sharded_offsets: tuple = (), metadata: Optional[Dict] = None + ) -> ShardedStateDict: + """Sharded state dict implementation for GPTModel backward-compatibility (removing extra state). + + Args: + prefix (str): Module name prefix. + sharded_offsets (tuple): PP related offsets, expected to be empty at this module level. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. + + Returns: + ShardedStateDict: sharded state dict for the GPTModel + """ + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) + + # Allow for shape mismatch in camera control spatial embedder for fine-tuning + if hasattr(self.config, "model_name"): + if "camera_ctrl" in self.config.model_name: + sharded_state_dict["module.x_embedder.proj.1.weight"].allow_shape_mismatch = True + + for param_name, param in self.t_embedder.named_parameters(): + weight_key = f"{prefix}t_embedder.{param_name}" + self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + if self.additional_timestamp_channels: + for cond_name, embedder in self.additional_timestamp_embedder.items(): + for param_name, param in embedder.named_parameters(): + weight_key = f"{prefix}additional_t_embedder_{cond_name}.{param_name}" + self.tie_embeddings_weights_state_dict(param, sharded_state_dict, weight_key, weight_key) + + return sharded_state_dict + + def tie_embeddings_weights_state_dict( + self, + tensor, + sharded_state_dict: ShardedStateDict, + output_layer_weight_key: str, + first_stage_word_emb_key: str, + ) -> None: + """Ties the embedding and output weights in a given sharded state dict. + + Args: + sharded_state_dict (ShardedStateDict): state dict with the weight to tie + output_layer_weight_key (str): key of the output layer weight in the state dict. + This entry will be replaced with a tied version + first_stage_word_emb_key (str): this must be the same as the + ShardedTensor.key of the first stage word embeddings. + + Returns: None, acts in-place + """ + if self.pre_process and parallel_state.get_tensor_model_parallel_rank() == 0: + # Output layer is equivalent to the embedding already + return + + # Replace the default output layer with a one sharing the weights with the embedding + del sharded_state_dict[output_layer_weight_key] + last_stage_word_emb_replica_id = ( + 0, # copy of first stage embedding + parallel_state.get_tensor_model_parallel_rank() + + parallel_state.get_pipeline_model_parallel_rank() + * parallel_state.get_pipeline_model_parallel_world_size(), + parallel_state.get_data_parallel_rank(with_context_parallel=True), + ) + + sharded_state_dict[output_layer_weight_key] = make_sharded_tensor_for_checkpoint( + tensor=tensor, + key=first_stage_word_emb_key, + replica_id=last_stage_word_emb_replica_id, + allow_shape_mismatch=False, + ) + + +class DiTCrossAttentionActionExtendModel7B(DiTCrossAttentionExtendModel7B): + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + rotary_percent: float = 1.0, + seq_len_interpolation_factor: Optional[float] = None, + ): + # Initialize base V2W model. + super().__init__( + config, + pre_process, + post_process, + fp16_lm_cross_entropy, + parallel_output, + position_embedding_type, + rotary_percent, + seq_len_interpolation_factor, + ) + + self.action_mlp = ActionControlTorchMlp( + self.config.action_emb_dim, hidden_features=config.hidden_size, out_features=config.hidden_size + ) + self.action_mlp_3d = ActionControlTorchMlp( + self.config.action_emb_dim, + hidden_features=config.hidden_size, + out_features=config.hidden_size, + action_3d=True, + ) + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + action_control_condition: Tensor = None, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + **kwargs, + ): + """Forward pass for the action-control V2W DiT model. + + Args: + x (Tensor): Tokenized video input of shape (B, C, T, H, W). + timestemps (Tensor): Timestep input tensor. + crossattn_emb (Tensor): Text embedding input for the cross-attention block. + action_control_condition (Tensor): Action control vector of shape (B, A), + where A is the action control dimensionality, e.g. A=7 for the bridge dataset. + + Returns: + Tensor: loss / noise prediction Tensor + """ + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + + input_list = [x, condition_video_input_mask] + x = torch.cat( + input_list, + dim=1, + ) + + rope_emb_L_1_1_D = None + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + B, T, H, W, D = x_B_T_H_W_D.shape + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # logging affline scale information + affline_scale_log_info = {} + + # Compute affine embeddings from the timesteps. + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + # Compute the action embedding MLP. + if action_control_condition is not None: + # Timestamp Embedding + action_embed = self.action_mlp(action_control_condition) + # Adaptive Layer Norm Embedding + action_embed_3d = self.action_mlp_3d(action_control_condition) + # Add action conditioning to affine embedding. + affline_emb_B_D = affline_emb_B_D + action_embed + if adaln_lora_B_3D is not None: + adaln_lora_B_3D = adaln_lora_B_3D + action_embed_3d + + # Setup additional channels and aggregate into affline_emb_B_D. + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + + # Run the TransformerBlock cross-attention decoder. attention_mask is re-purposed to pass in the affine embedding. + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + packed_seq_params=packed_seq_params, + rotary_pos_emb=rope_emb_L_1_1_D[0], + ) + + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + # Returns (B, T, H, W, D) video prediction. + return x_B_T_H_W_D + + +class DiTControl7B(DiTCrossAttentionExtendModel7B): + """ + DiTControlNet builds on top of DiTCrossAttentionExtendModel7B and mirrors the structure of repo1's ControlNet. + + It creates a control branch (self.ctrl_net) that processes an encoded control input ("hint") and then + uses a condition indicator mask to blend the control branch output with the original input. + + The methods get_data_and_condition and encode_latent are assumed to be provided externally. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dropout_ctrl_branch = kwargs.pop("dropout_ctrl_branch", 0.5) + hint_channels = kwargs.pop("hint_channels", 128) + num_control_blocks = kwargs.pop("num_control_blocks", 3) + num_blocks = self.config.num_layers + + if num_control_blocks is not None: + assert num_control_blocks > 0 and num_control_blocks <= num_blocks + kwargs["layer_mask"] = [False] * num_control_blocks + [True] * (num_blocks - num_control_blocks) + self.random_drop_control_blocks = kwargs.pop("random_drop_control_blocks", False) + model_channels = self.config.hidden_size + self.model_channels = model_channels + layer_mask = kwargs.get("layer_mask", None) + layer_mask = [False] * num_blocks if layer_mask is None else layer_mask + self.layer_mask = layer_mask + self.hint_channels = hint_channels + self.build_hint_patch_embed() + hint_nf = [16, 16, 32, 32, 96, 96, 256] + nonlinearity = nn.SiLU() + input_hint_block = [nn.Linear(model_channels, hint_nf[0]), nonlinearity] + for i in range(len(hint_nf) - 1): + input_hint_block += [nn.Linear(hint_nf[i], hint_nf[i + 1]), nonlinearity] + self.input_hint_block = nn.Sequential(*input_hint_block) + # Initialize weights + self.zero_blocks = nn.ModuleDict() + for idx in range(num_blocks): + if layer_mask[idx]: + continue + blk = nn.Linear(model_channels, model_channels) + self.zero_blocks[f"block{idx}"] = blk + blk = nn.Linear(hint_nf[-1], model_channels) + self.input_hint_block.append(blk) + + def build_hint_patch_embed(self): + concat_padding_mask, in_channels, patch_spatial, patch_temporal, model_channels = ( + self.concat_padding_mask, + self.hint_channels, + self.patch_spatial, + self.patch_temporal, + self.model_channels, + ) + in_channels = in_channels + 1 if concat_padding_mask else in_channels + self.x_embedder2 = PatchEmbed( + spatial_patch_size=patch_spatial, + temporal_patch_size=patch_temporal, + in_channels=in_channels, + out_channels=model_channels, + bias=False, + keep_spatio=True, + legacy_patch_emb=False, + ) + + def prepare_hint_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + if self.concat_padding_mask: + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(x_B_C_T_H_W.shape[0], 1, x_B_C_T_H_W.shape[2], 1, 1)], + dim=1, + ) + + x_B_T_H_W_D = self.x_embedder2(x_B_C_T_H_W) + + if "rope" in self.pos_emb_cls.lower(): + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps) + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None + + def encode_hint( + self, + hint: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + ) -> torch.Tensor: + assert hint.size(1) <= self.hint_channels, ( + f"Expected hint channels <= {self.hint_channels}, got {hint.size(1)}" + ) + if hint.size(1) < self.hint_channels: + padding_shape = list(hint.shape) + padding_shape[1] = self.hint_channels - hint.size(1) + hint = torch.cat([hint, torch.zeros(*padding_shape, dtype=hint.dtype, device=hint.device)], dim=1) + assert isinstance(data_type, DataType), ( + f"Expected DataType, got {type(data_type)}. We need discuss this flag later." + ) + + hint_B_T_H_W_D, _ = self.prepare_hint_embedded_sequence(hint, fps=fps, padding_mask=padding_mask) + + hint = hint_B_T_H_W_D + + guided_hint = self.input_hint_block(hint) + return guided_hint + + def encode_latent(self, data_batch: dict, tokenizer, cond_mask: list = []) -> torch.Tensor: + x = data_batch[data_batch["hint_key"]] + latent = [] + # control input goes through tokenizer, which always takes 3-input channels + num_conditions = x.size(1) // 3 # input conditions were concatenated along channel dimension + if num_conditions > 1 and self.config.hint_dropout_rate > 0: + if torch.is_grad_enabled(): # during training, randomly dropout some conditions + cond_mask = torch.rand(num_conditions) > self.config.hint_dropout_rate + if not cond_mask.any(): # make sure at least one condition is present + cond_mask = [True] * num_conditions + elif not cond_mask: # during inference, use hint_mask to indicate which conditions are used + cond_mask = self.config.hint_mask + else: + cond_mask = [True] * num_conditions + for idx in range(0, x.size(1), 3): + x_rgb = x[:, idx : idx + 3] + if not cond_mask[idx // 3]: # if the condition is not selected, replace with a black image + x_rgb = torch.zeros_like(x_rgb) + latent.append(tokenizer.encode(x_rgb) * self.sigma_data) + latent = torch.cat(latent, dim=1) + return latent + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + data_type: Optional[DataType] = DataType.VIDEO, + hint_key: Optional[str] = None, + base_model: Optional[nn.Module] = None, + control_weight: Optional[float] = 1.0, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + x_input = x.clone() + crossattn_emb_input = crossattn_emb + condition_video_input_mask_input = condition_video_input_mask + fps = kwargs.get("fps", None) + padding_mask = kwargs.get("padding_mask", None) + hint = kwargs.get(hint_key) + crossattn_mask = kwargs.get("crossattn_mask", None) + + if hint is None: + return base_model.module.module.module.forward( + x=x_input, + timesteps=timesteps, + crossattn_emb=crossattn_emb_input, + inference_params=inference_params, + packed_seq_params=packed_seq_params, + pos_ids=pos_ids, + condition_video_input_mask=condition_video_input_mask_input, + **kwargs, + ) + guided_hints = self.encode_hint(hint, fps=fps, padding_mask=padding_mask, data_type=data_type) + + guided_hints = torch.chunk(guided_hints, hint.shape[0] // x.shape[0], dim=3) + assert isinstance(data_type, DataType), ( + f"Expected DataType, got {type(data_type)}. We need discuss this flag later." + ) + B, C, T, H, W = x_input.shape + if data_type == DataType.VIDEO: + if condition_video_input_mask is not None: + input_list = [x, condition_video_input_mask] + x = torch.cat(input_list, dim=1) + + elif data_type == DataType.IMAGE: + # For image, we dont have condition_video_input_mask, or condition_video_pose + # We need to add the extra channel for video condition mask + padding_channels = self.in_channels - x.shape[1] + x = torch.cat([x, torch.zeros((B, padding_channels, T, H, W), dtype=x.dtype, device=x.device)], dim=1) + else: + assert x.shape[1] == self.in_channels, f"Expected {self.in_channels} channels, got {x.shape[1]}" + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence(x, fps=fps, padding_mask=padding_mask) + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if scalar_feature is not None: + raise NotImplementedError("Scalar feature is not implemented yet.") + timesteps_B_D = timesteps_B_D + scalar_feature.mean(dim=1) + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + # for logging purpose + self.affline_scale_log_info = affline_scale_log_info + self.affline_emb = affline_emb_B_D + self.crossattn_emb = crossattn_emb + self.crossattn_mask = crossattn_mask + + # if self.use_cross_attn_mask: + # crossattn_mask = crossattn_mask[:, None, None, :].to(dtype=torch.bool) # [B, 1, 1, length] + # else: + crossattn_mask = None + + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + x = x_S_B_D + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + outs = {} + + # If also training base model, sometimes drop the controlnet branch to only train base branch. + # This is to prevent the network become dependent on controlnet branch and make control weight useless. + is_training = torch.is_grad_enabled() + is_training_base_model = any(p.requires_grad for p in base_model.parameters()) + if is_training and is_training_base_model: + coin_flip = torch.rand(B).to(x.device) > self.dropout_ctrl_branch # prob for only training base model + coin_flip = coin_flip[:, None, None, None, None] + else: + coin_flip = 1 + + num_control_blocks = self.layer_mask.index(True) + if self.random_drop_control_blocks: + if is_training: # Use a random number of layers during training. + num_layers_to_use = np.random.randint(num_control_blocks) + 1 + elif num_layers_to_use == -1: # Evaluate using all the layers. + num_layers_to_use = num_control_blocks + else: # Use the specified number of layers during inference. + pass + else: # Use all of the layers. + num_layers_to_use = num_control_blocks + + if isinstance(control_weight, torch.Tensor) and control_weight.shape[0] > 1: + pass + else: + control_weight = [control_weight] * len(guided_hints) + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + for i, guided_hint in enumerate(guided_hints): + for idx, block in enumerate(self.decoder.layers): + name = f"block{idx}" + x, _ = block( + x, + affline_emb_B_D, + crossattn_emb, + None, + rotary_pos_emb=rope_emb_L_1_1_D[0], + packed_seq_params=packed_seq_params, + ) + if guided_hint is not None: + gh_S_B_D = rearrange(guided_hint, "B T H W D -> (T H W) B D") + x = x + gh_S_B_D + gh_S_B_D = None + guided_hint = None + + gate = idx < num_control_blocks + # x_B_T_H_W_D = rearrange(x, '(T H W) B D -> B T H W D', T=T, H=H, W=W) + if gate: + hint_val = self.zero_blocks[name](x) * control_weight[i] * coin_flip * gate + if name not in outs: + outs[name] = hint_val + else: + outs[name] += hint_val + output = base_model.module.module.module.forward( + x=x_input, + x_ctrl=outs, + timesteps=timesteps, + crossattn_emb=crossattn_emb_input, + inference_params=inference_params, + packed_seq_params=packed_seq_params, + pos_ids=pos_ids, + condition_video_input_mask=condition_video_input_mask_input, + **kwargs, + ) + return output + + +# flake8: noqa: E741 diff --git a/nemo_vfm/diffusion/models/dit_llama/__init__.py b/nemo_vfm/diffusion/models/dit_llama/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit_llama/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/models/dit_llama/dit_llama_layer_spec.py b/nemo_vfm/diffusion/models/dit_llama/dit_llama_layer_spec.py new file mode 100644 index 00000000..8967ab6e --- /dev/null +++ b/nemo_vfm/diffusion/models/dit_llama/dit_llama_layer_spec.py @@ -0,0 +1,175 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import copy +from typing import Literal + +from megatron.core.transformer.attention import ( + CrossAttention, + CrossAttentionSubmodules, + SelfAttention, + SelfAttentionSubmodules, +) +from megatron.core.transformer.custom_layers.transformer_engine import ( + TEColumnParallelLinear, + TEDotProductAttention, + TENorm, + TERowParallelLinear, +) +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_block import TransformerConfig +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from megatron.core.utils import make_viewless_tensor +from nemo.collections.diffusion.models.dit.dit_layer_spec import AdaLN + + +class MoviegGenLayer(TransformerLayer): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + + DiT with Adapative Layer Normalization. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + hidden_dropout: float = None, + position_embedding_type: Literal["learned_absolute", "rope"] = "learned_absolute", + ): + def _replace_no_cp_submodules(submodules): + modified_submods = copy.deepcopy(submodules) + modified_submods.cross_attention = IdentityOp + # modified_submods.temporal_self_attention = IdentityOp + return modified_submods + + # Replace any submodules that will have CP disabled and build them manually later after TransformerLayer init. + modified_submods = _replace_no_cp_submodules(submodules) + super().__init__( + config=config, submodules=modified_submods, layer_number=layer_number, hidden_dropout=hidden_dropout + ) + + # Override Cross Attention to disable CP. + # Disable TP Comm overlap as well. Not disabling will attempt re-use of buffer size same as Q and lead to incorrect tensor shapes. + cp_override_config = copy.deepcopy(config) + cp_override_config.context_parallel_size = 1 + cp_override_config.tp_comm_overlap = False + self.cross_attention = build_module( + submodules.cross_attention, + config=cp_override_config, + layer_number=layer_number, + ) + + self.adaLN = AdaLN(config=self.config, n_adaln_chunks=6, norm=TENorm) + + def forward( + self, + hidden_states, + attention_mask, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + ): + # timestep embedding + timestep_emb = attention_mask + factorized_pos_emb = rotary_pos_emb + hidden_states = hidden_states + factorized_pos_emb + + # ******************************************** full self attention ****************************************************** + shift_full, scale_full, gate_full, shift_mlp, scale_mlp, gate_mlp = self.adaLN(timestep_emb) + + # adaLN with scale + shift + pre_full_attn_layernorm_output_ada = self.adaLN.modulated_layernorm( + hidden_states, shift=shift_full, scale=scale_full + ) + + attention_output, _ = self.self_attention( + pre_full_attn_layernorm_output_ada, + attention_mask=None, + packed_seq_params=None if packed_seq_params is None else packed_seq_params["self_attention"], + ) + + hidden_states = self.adaLN.scale_add(residual=hidden_states, x=attention_output, gate=gate_full) + + # ******************************************** cross attention ****************************************************** + attention_output, _ = self.cross_attention( + hidden_states, + attention_mask=context_mask, + key_value_states=context, + packed_seq_params=None if packed_seq_params is None else packed_seq_params["cross_attention"], + ) + + # ******************************************** mlp ****************************************************** + pre_mlp_layernorm_output_ada = self.adaLN.modulated_layernorm( + attention_output, shift=shift_mlp, scale=scale_mlp + ) + + mlp_output, _ = self.mlp(pre_mlp_layernorm_output_ada) + hidden_states = self.adaLN.scale_add(residual=hidden_states, x=mlp_output, gate=gate_mlp) + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + output = make_viewless_tensor(inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True) + + return output, context + + +def get_dit_llama_spec() -> ModuleSpec: + params = {"attn_mask_type": AttnMaskType.padding} + return ModuleSpec( + module=MoviegGenLayer, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=SelfAttention, + params=params, + submodules=SelfAttentionSubmodules( + linear_qkv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + ), + ), + cross_attention=ModuleSpec( + module=CrossAttention, + params=params, + submodules=CrossAttentionSubmodules( + linear_q=TEColumnParallelLinear, + linear_kv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + ), + ), + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TEColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + ), + ) diff --git a/nemo_vfm/diffusion/models/dit_llama/dit_llama_model.py b/nemo_vfm/diffusion/models/dit_llama/dit_llama_model.py new file mode 100644 index 00000000..827eb270 --- /dev/null +++ b/nemo_vfm/diffusion/models/dit_llama/dit_llama_model.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + + +from typing import Literal + +from megatron.core.transformer.transformer_config import TransformerConfig +from nemo.collections.diffusion.models.dit import dit_embeddings +from nemo.collections.diffusion.models.dit.dit_model import DiTCrossAttentionModel +from nemo.collections.diffusion.models.dit_llama.dit_llama_layer_spec import get_dit_llama_spec + + +class DiTLlamaModel(DiTCrossAttentionModel): + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + max_img_h: int = 80, + max_img_w: int = 80, + max_frames: int = 34, + patch_spatial: int = 1, + patch_temporal: int = 1, + in_channels: int = 16, + out_channels: int = 16, + **kwargs, + ): + super().__init__( + config=config, + pre_process=pre_process, + post_process=post_process, + fp16_lm_cross_entropy=fp16_lm_cross_entropy, + parallel_output=parallel_output, + position_embedding_type=position_embedding_type, + max_img_h=max_img_h, + max_img_w=max_img_w, + max_frames=max_frames, + patch_spatial=patch_spatial, + patch_temporal=patch_temporal, + in_channels=in_channels, + out_channels=out_channels, + transformer_decoder_layer_spec=get_dit_llama_spec, + pos_embedder=dit_embeddings.FactorizedLearnable3DEmbedding, + **kwargs, + ) diff --git a/nemo_vfm/diffusion/models/flux/__init__.py b/nemo_vfm/diffusion/models/flux/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/models/flux/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/models/flux/layers.py b/nemo_vfm/diffusion/models/flux/layers.py new file mode 100644 index 00000000..d47454f3 --- /dev/null +++ b/nemo_vfm/diffusion/models/flux/layers.py @@ -0,0 +1,215 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math + +import torch +from torch import Tensor, nn + + +def rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor: + """ + Different from the original ROPE used for flux. + Megatron attention takes the out product and calculate sin/cos inside, so we only need to get the freqs here + in the shape of [seq, ..., dim] + """ + assert dim % 2 == 0, "The dimension must be even." + + scale = torch.arange(0, dim, 2, dtype=torch.float64, device=pos.device) / dim + omega = 1.0 / (theta**scale) + + out = torch.einsum("...n,d->...nd", pos, omega) + + return out.float() + + +class EmbedND(nn.Module): + """ + Generate Rope matrix with preset axes dimensions. + """ + + def __init__(self, dim: int, theta: int, axes_dim: list[int]): + # pylint: disable=C0116 + super().__init__() + self.dim = dim + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + # pylint: disable=C0116 + n_axes = ids.shape[-1] + emb = torch.cat( + [rope(ids[..., i], self.axes_dim[i], self.theta) for i in range(n_axes)], + dim=-1, + ) + emb = emb.unsqueeze(1).permute(2, 0, 1, 3) + return torch.stack([emb, emb], dim=-1).reshape(*emb.shape[:-1], -1) + + +class MLPEmbedder(nn.Module): + """ + MLP embedder with two projection layers and Silu in between. + """ + + def __init__(self, in_dim: int, hidden_dim: int): + # pylint: disable=C0116 + super().__init__() + self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True) + self.silu = nn.SiLU() + self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def forward(self, x: Tensor) -> Tensor: + # pylint: disable=C0116 + return self.out_layer(self.silu(self.in_layer(x))) + + +def get_timestep_embedding( + timesteps: torch.Tensor, + embedding_dim: int, + flip_sin_to_cos: bool = True, + downscale_freq_shift: float = 0, + scale: float = 1, + max_period: int = 10000, +): + """ + This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings. + + Args + timesteps (torch.Tensor): + a 1-D Tensor of N indices, one per batch element. These may be fractional. + embedding_dim (int): + the dimension of the output. + flip_sin_to_cos (bool): + Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False) + downscale_freq_shift (float): + Controls the delta between frequencies between dimensions + scale (float): + Scaling factor applied to the embeddings. + max_period (int): + Controls the maximum frequency of the embeddings + Returns + torch.Tensor: an [N x dim] Tensor of positional embeddings. + """ + assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array" + + half_dim = embedding_dim // 2 + exponent = -math.log(max_period) * torch.arange( + start=0, end=half_dim, dtype=torch.float32, device=timesteps.device + ) + exponent = exponent / (half_dim - downscale_freq_shift) + + emb = torch.exp(exponent) + emb = timesteps[:, None].float() * emb[None, :] + + # scale embeddings + emb = scale * emb + + # concat sine and cosine embeddings + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1) + + # flip sine and cosine embeddings + if flip_sin_to_cos: + emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1) + + # zero pad + if embedding_dim % 2 == 1: + emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) + return emb + + +# pylint: disable=C0116 +class Timesteps(nn.Module): + def __init__( + self, + embedding_dim: int, + flip_sin_to_cos: bool = True, + downscale_freq_shift: float = 0, + scale: float = 1, + max_period: int = 10000, + ): + super().__init__() + self.embedding_dim = embedding_dim + self.flip_sin_to_cos = flip_sin_to_cos + self.downscale_freq_shift = downscale_freq_shift + self.scale = scale + self.max_period = max_period + + def forward(self, timesteps: torch.Tensor) -> torch.Tensor: + t_emb = get_timestep_embedding( + timesteps, + self.embedding_dim, + flip_sin_to_cos=self.flip_sin_to_cos, + downscale_freq_shift=self.downscale_freq_shift, + scale=self.scale, + max_period=self.max_period, + ) + return t_emb + + +# pylint: disable=C0116 + + +class TimeStepEmbedder(nn.Module): + """ + A neural network module that embeds timesteps for use in models such as diffusion models. + It projects the input timesteps to a higher-dimensional space and then embeds them using + an MLP (Multilayer Perceptron). The projection and embedding provide a learned representation + of the timestep that can be used in further computations. + + Args: + embedding_dim (int): + The dimensionality of the timestep embedding space. + hidden_dim (int): + The dimensionality of the hidden layer in the MLPEmbedder. + flip_sin_to_cos (bool, optional): + Whether to flip the sine and cosine components during the projection (default is True). + downscale_freq_shift (float, optional): + A scaling factor for the frequency shift during the projection (default is 0). + scale (float, optional): + A scaling factor applied to the timestep projections (default is 1). + max_period (int, optional): + The maximum period for the sine and cosine functions used in projection (default is 10000). + + Methods: + forward: Takes a tensor of timesteps and returns their embedded representation. + """ + + def __init__( + self, + embedding_dim: int, + hidden_dim: int, + flip_sin_to_cos: bool = True, + downscale_freq_shift: float = 0, + scale: float = 1, + max_period: int = 10000, + ): + super().__init__() + + self.time_proj = Timesteps( + embedding_dim=embedding_dim, + flip_sin_to_cos=flip_sin_to_cos, + downscale_freq_shift=downscale_freq_shift, + scale=scale, + max_period=max_period, + ) + self.time_embedder = MLPEmbedder(in_dim=embedding_dim, hidden_dim=hidden_dim) + + def forward(self, timesteps: torch.Tensor) -> torch.Tensor: + # pylint: disable=C0116 + timesteps_proj = self.time_proj(timesteps) + timesteps_emb = self.time_embedder(timesteps_proj) + + return timesteps_emb diff --git a/nemo_vfm/diffusion/models/flux/model.py b/nemo_vfm/diffusion/models/flux/model.py new file mode 100644 index 00000000..fb0d7d30 --- /dev/null +++ b/nemo_vfm/diffusion/models/flux/model.py @@ -0,0 +1,931 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +import lightning.pytorch as L +import numpy as np +import torch +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding +from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.optimizer import OptimizerConfig +from megatron.core.transformer.enums import ModelType +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.utils import openai_gelu, sharded_state_dict_default +from nemo.collections.diffusion.encoders.conditioner import FrozenCLIPEmbedder, FrozenT5Embedder +from nemo.collections.diffusion.models.dit.dit_layer_spec import ( + AdaLNContinuous, + FluxSingleTransformerBlock, + MMDiTLayer, + get_flux_double_transformer_engine_spec, + get_flux_single_transformer_engine_spec, +) +from nemo.collections.diffusion.models.flux.layers import EmbedND, MLPEmbedder, TimeStepEmbedder +from nemo.collections.diffusion.sampler.flow_matching.flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler +from nemo.collections.diffusion.utils.flux_ckpt_converter import ( + _import_qkv, + _import_qkv_bias, + flux_transformer_converter, +) +from nemo.collections.diffusion.vae.autoencoder import AutoEncoder, AutoEncoderConfig +from nemo.collections.llm import fn +from nemo.lightning import io, teardown +from nemo.lightning.megatron_parallel import MaskedTokenLossReduction +from nemo.lightning.pytorch.optim import MegatronOptimizerModule, OptimizerModule +from nemo.utils import logging +from safetensors.torch import load_file as load_safetensors +from safetensors.torch import save_file as save_safetensors +from torch import nn +from torch.nn import functional as F + + +# pylint: disable=C0116 +def flux_data_step(dataloader_iter): + batch = next(dataloader_iter) + if isinstance(batch, tuple) and len(batch) == 3: + _batch = batch[0] + else: + _batch = batch + + _batch["loss_mask"] = torch.Tensor([1.0]).cuda(non_blocking=True) + return _batch + + +@dataclass +class FluxConfig(TransformerConfig, io.IOMixin): + """ + transformer related Flux Config + """ + + num_layers: int = 1 # dummy setting + num_joint_layers: int = 19 + num_single_layers: int = 38 + hidden_size: int = 3072 + num_attention_heads: int = 24 + activation_func: Callable = openai_gelu + add_qkv_bias: bool = True + in_channels: int = 64 + context_dim: int = 4096 + model_channels: int = 256 + patch_size: int = 1 + guidance_embed: bool = False + vec_in_dim: int = 768 + rotary_interleaved: bool = True + apply_rope_fusion: bool = False + layernorm_epsilon: float = 1e-06 + hidden_dropout: float = 0 + attention_dropout: float = 0 + use_cpu_initialization: bool = True + gradient_accumulation_fusion: bool = True + enable_cuda_graph: bool = False + use_te_rng_tracker: bool = False + cuda_graph_warmup_steps: int = 2 + + guidance_scale: float = 3.5 + data_step_fn: Callable = flux_data_step + ckpt_path: Optional[str] = None + load_dist_ckpt: bool = False + do_convert_from_hf: bool = False + save_converted_model_to = None + + def configure_model(self): + model = Flux(config=self) + return model + + +@dataclass +class T5Config: + """ + T5 Config + """ + + version: Optional[str] = field(default_factory=lambda: "google/t5-v1_1-xxl") + max_length: Optional[int] = field(default_factory=lambda: 512) + + +@dataclass +class ClipConfig: + """ + Clip Config + """ + + version: Optional[str] = field(default_factory=lambda: "openai/clip-vit-large-patch14") + max_length: Optional[int] = field(default_factory=lambda: 77) + always_return_pooled: Optional[bool] = field(default_factory=lambda: True) + + +@dataclass +class FluxModelParams: + """ + Flux Model Params + """ + + flux_config: FluxConfig = field(default_factory=FluxConfig) + vae_config: AutoEncoderConfig = field( + default_factory=lambda: AutoEncoderConfig(ch_mult=[1, 2, 4, 4], attn_resolutions=[]) + ) + clip_params: ClipConfig = field(default_factory=ClipConfig) + t5_params: T5Config = field(default_factory=T5Config) + + scheduler_steps: int = 1000 + device: str = "cuda" + + +# pylint: disable=C0116 +class Flux(VisionModule): + """ + NeMo implementation of Flux model, with flux transformer and single flux transformer blocks implemented with + Megatron Core. + + Args: + config (FluxConfig): Configuration object containing the necessary parameters for setting up the model, + such as the number of channels, hidden size, attention heads, and more. + + Attributes: + out_channels (int): The number of output channels for the model. + hidden_size (int): The size of the hidden layers. + num_attention_heads (int): The number of attention heads for the transformer. + patch_size (int): The size of the image patches. + in_channels (int): The number of input channels for the image. + guidance_embed (bool): A flag to indicate if guidance embedding should be used. + pos_embed (EmbedND): Position embedding layer for the model. + img_embed (nn.Linear): Linear layer to embed image input into the hidden space. + txt_embed (nn.Linear): Linear layer to embed text input into the hidden space. + timestep_embedding (TimeStepEmbedder): Embedding layer for timesteps, used in generative models. + vector_embedding (MLPEmbedder): MLP embedding for vector inputs. + guidance_embedding (nn.Module or nn.Identity): Optional MLP embedding for guidance, or identity if not used. + double_blocks (nn.ModuleList): A list of transformer blocks for the double block layers. + single_blocks (nn.ModuleList): A list of transformer blocks for the single block layers. + norm_out (AdaLNContinuous): Normalization layer for the output. + proj_out (nn.Linear): Final linear layer for output projection. + + Methods: + forward: Performs a forward pass through the network, processing images, text, timesteps, and guidance. + load_from_pretrained: Loads model weights from a pretrained checkpoint, with optional support for distribution + and conversion from Hugging Face format. + + """ + + def __init__(self, config: FluxConfig): + # pylint: disable=C0116 + super().__init__(config) + self.out_channels = config.in_channels + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.patch_size = config.patch_size + self.in_channels = config.in_channels + self.guidance_embed = config.guidance_embed + + self.pos_embed = EmbedND(dim=self.hidden_size, theta=10000, axes_dim=[16, 56, 56]) + self.img_embed = nn.Linear(config.in_channels, self.hidden_size) + self.txt_embed = nn.Linear(config.context_dim, self.hidden_size) + self.timestep_embedding = TimeStepEmbedder(config.model_channels, self.hidden_size) + self.vector_embedding = MLPEmbedder(in_dim=config.vec_in_dim, hidden_dim=self.hidden_size) + if config.guidance_embed: + self.guidance_embedding = ( + MLPEmbedder(in_dim=config.model_channels, hidden_dim=self.hidden_size) + if config.guidance_embed + else nn.Identity() + ) + + self.double_blocks = nn.ModuleList( + [ + MMDiTLayer( + config=config, + submodules=get_flux_double_transformer_engine_spec().submodules, + layer_number=i, + context_pre_only=False, + ) + for i in range(config.num_joint_layers) + ] + ) + + self.single_blocks = nn.ModuleList( + [ + FluxSingleTransformerBlock( + config=config, + submodules=get_flux_single_transformer_engine_spec().submodules, + layer_number=i, + ) + for i in range(config.num_single_layers) + ] + ) + + self.norm_out = AdaLNContinuous(config=config, conditioning_embedding_dim=self.hidden_size) + self.proj_out = nn.Linear(self.hidden_size, self.patch_size * self.patch_size * self.out_channels, bias=True) + if self.config.ckpt_path is not None: + self.load_from_pretrained( + self.config.ckpt_path, + do_convert_from_hf=self.config.do_convert_from_hf, + load_dist_ckpt=self.config.load_dist_ckpt, + save_converted_model_to=self.config.save_converted_model_to, + ) + + def forward( + self, + img: torch.Tensor, + txt: torch.Tensor = None, + y: torch.Tensor = None, + timesteps: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + controlnet_double_block_samples: torch.Tensor = None, + controlnet_single_block_samples: torch.Tensor = None, + ): + """ + Forward pass through the model, processing image, text, and additional inputs like guidance and timesteps. + + Args: + img (torch.Tensor): + The image input tensor. + txt (torch.Tensor, optional): + The text input tensor (default is None). + y (torch.Tensor, optional): + The vector input for embedding (default is None). + timesteps (torch.LongTensor, optional): + The timestep input, typically used in generative models (default is None). + img_ids (torch.Tensor, optional): + Image IDs for positional encoding (default is None). + txt_ids (torch.Tensor, optional): + Text IDs for positional encoding (default is None). + guidance (torch.Tensor, optional): + Guidance input for conditioning (default is None). + controlnet_double_block_samples (torch.Tensor, optional): + Optional controlnet samples for double blocks (default is None). + controlnet_single_block_samples (torch.Tensor, optional): + Optional controlnet samples for single blocks (default is None). + + Returns: + torch.Tensor: The final output tensor from the model after processing all inputs. + """ + hidden_states = self.img_embed(img) + encoder_hidden_states = self.txt_embed(txt) + + timesteps = timesteps.to(img.dtype) * 1000 + vec_emb = self.timestep_embedding(timesteps) + + if guidance is not None: + vec_emb = vec_emb + self.guidance_embedding(self.timestep_embedding.time_proj(guidance * 1000)) + vec_emb = vec_emb + self.vector_embedding(y) + + ids = torch.cat((txt_ids, img_ids), dim=1) + rotary_pos_emb = self.pos_embed(ids) + for id_block, block in enumerate(self.double_blocks): + hidden_states, encoder_hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + rotary_pos_emb=rotary_pos_emb, + emb=vec_emb, + ) + + if controlnet_double_block_samples is not None: + interval_control = len(self.double_blocks) / len(controlnet_double_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states = hidden_states + controlnet_double_block_samples[id_block // interval_control] + + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=0) + + for id_block, block in enumerate(self.single_blocks): + hidden_states, _ = block( + hidden_states=hidden_states, + rotary_pos_emb=rotary_pos_emb, + emb=vec_emb, + ) + + if controlnet_single_block_samples is not None: + interval_control = len(self.single_blocks) / len(controlnet_single_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states = torch.cat( + [ + hidden_states[: encoder_hidden_states.shape[0]], + hidden_states[encoder_hidden_states.shape[0] :] + + controlnet_single_block_samples[id_block // interval_control], + ] + ) + + hidden_states = hidden_states[encoder_hidden_states.shape[0] :, ...] + + hidden_states = self.norm_out(hidden_states, vec_emb) + output = self.proj_out(hidden_states) + + return output + + def load_from_pretrained( + self, ckpt_path, do_convert_from_hf=False, save_converted_model_to=None, load_dist_ckpt=False + ): + # pylint: disable=C0116 + if load_dist_ckpt: + from megatron.core import dist_checkpointing + + sharded_state_dict = dict(state_dict=self.sharded_state_dict(prefix="module.")) + loaded_state_dict = dist_checkpointing.load( + sharded_state_dict=sharded_state_dict, checkpoint_dir=ckpt_path + ) + ckpt = {k.removeprefix("module."): v for k, v in loaded_state_dict["state_dict"].items()} + else: + if do_convert_from_hf: + ckpt = flux_transformer_converter(ckpt_path, self.config) + if save_converted_model_to is not None: + os.makedirs(save_converted_model_to, exist_ok=True) + save_path = os.path.join(save_converted_model_to, "nemo_flux_transformer.safetensors") + save_safetensors(ckpt, save_path) + logging.info(f"saving converted transformer checkpoint to {save_path}") + else: + ckpt = load_safetensors(ckpt_path) + missing, unexpected = self.load_state_dict(ckpt, strict=False) + missing = [k for k in missing if not k.endswith("_extra_state")] + # These keys are mcore specific and should not affect the model performance + if len(missing) > 0: + logging.info( + f"The following keys are missing during checkpoint loading, " + f"please check the ckpt provided or the image quality may be compromised.\n {missing}" + ) + logging.info(f"Found unexepected keys: \n {unexpected}") + logging.info(f"Restored flux model weights from {ckpt_path}") + + def sharded_state_dict(self, prefix="", sharded_offsets: tuple = (), metadata: dict = None) -> ShardedStateDict: + sharded_state_dict = {} + layer_prefix = f"{prefix}double_blocks." + for layer in self.double_blocks: + offset = layer._get_layer_offset(self.config) + + global_layer_offset = layer.layer_number + state_dict_prefix = f"{layer_prefix}{global_layer_offset - offset}." + sharded_prefix = f"{layer_prefix}{global_layer_offset}." + sharded_pp_offset = [] + + layer_sharded_state_dict = layer.sharded_state_dict(state_dict_prefix, sharded_pp_offset, metadata) + replace_prefix_for_sharding(layer_sharded_state_dict, state_dict_prefix, sharded_prefix) + + sharded_state_dict.update(layer_sharded_state_dict) + + layer_prefix = f"{prefix}single_blocks." + for layer in self.single_blocks: + offset = layer._get_layer_offset(self.config) + + global_layer_offset = layer.layer_number + state_dict_prefix = f"{layer_prefix}{global_layer_offset - offset}." + sharded_prefix = f"{layer_prefix}{global_layer_offset}." + sharded_pp_offset = [] + + layer_sharded_state_dict = layer.sharded_state_dict(state_dict_prefix, sharded_pp_offset, metadata) + replace_prefix_for_sharding(layer_sharded_state_dict, state_dict_prefix, sharded_prefix) + + sharded_state_dict.update(layer_sharded_state_dict) + + for name, module in self.named_children(): + if not (module is self.single_blocks or module is self.double_blocks): + sharded_state_dict.update( + sharded_state_dict_default(module, f"{prefix}{name}.", sharded_offsets, metadata) + ) + return sharded_state_dict + + +class MegatronFluxModel(L.LightningModule, io.IOMixin, io.ConnectorMixin, fn.FNMixin): + """ + Megatron wrapper for flux. + + Args: + flux_params (FluxModelParams): Parameters to configure the Flux model. + """ + + def __init__( + self, + flux_params: FluxModelParams, + optim: Optional[OptimizerModule] = None, + ): + # pylint: disable=C0116 + self.params = flux_params + self.config = flux_params.flux_config + super().__init__() + self._training_loss_reduction = None + self._validation_loss_reduction = None + + self.vae_config = self.params.vae_config + self.clip_params = self.params.clip_params + self.t5_params = self.params.t5_params + self.optim = optim or MegatronOptimizerModule(config=OptimizerConfig(lr=1e-4, use_distributed_optimizer=False)) + self.optim.connect(self) + self.model_type = ModelType.encoder_or_decoder + self.text_precached = self.t5_params is None or self.clip_params is None + self.image_precached = self.vae_config is None + + def configure_model(self): + # pylint: disable=C0116 + if not hasattr(self, "module"): + self.module = self.config.configure_model() + self.configure_vae(self.vae_config) + self.configure_scheduler() + self.configure_text_encoders(self.clip_params, self.t5_params) + for name, param in self.module.named_parameters(): + if self.config.num_single_layers == 0: + if "context" in name or "added" in name: + param.requires_grad = False + # When getting rid of concat, the projection bias in attention and mlp bias are identical + # So this bias is skipped and not included in the computation graph + if "single_blocks" in name and "self_attention.linear_proj.bias" in name: + param.requires_grad = False + + def configure_scheduler(self): + # pylint: disable=C0116 + self.scheduler = FlowMatchEulerDiscreteScheduler( + num_train_timesteps=self.params.scheduler_steps, + ) + + def configure_vae(self, vae): + # pylint: disable=C0116 + if isinstance(vae, nn.Module): + self.vae = vae.eval().cuda() + self.vae_scale_factor = 2 ** (len(self.vae.params.ch_mult)) + for param in self.vae.parameters(): + param.requires_grad = False + elif isinstance(vae, AutoEncoderConfig): + self.vae = AutoEncoder(vae).eval().cuda() + self.vae_scale_factor = 2 ** (len(vae.ch_mult)) + for param in self.vae.parameters(): + param.requires_grad = False + else: + logging.info("Vae not provided, assuming the image input is precached...") + self.vae = None + self.vae_scale_factor = 16 + + def configure_text_encoders(self, clip, t5): + # pylint: disable=C0116 + if isinstance(clip, nn.Module): + self.clip = clip + elif isinstance(clip, ClipConfig): + self.clip = FrozenCLIPEmbedder( + version=self.clip_params.version, + max_length=self.clip_params.max_length, + always_return_pooled=self.clip_params.always_return_pooled, + device=torch.cuda.current_device(), + ) + else: + logging.info("CLIP encoder not provided, assuming the text embeddings is precached...") + self.clip = None + + if isinstance(t5, nn.Module): + self.t5 = t5 + elif isinstance(t5, T5Config): + self.t5 = FrozenT5Embedder( + self.t5_params.version, max_length=self.t5_params.max_length, device=torch.cuda.current_device() + ) + else: + logging.info("T5 encoder not provided, assuming the text embeddings is precached...") + self.t5 = None + + # pylint: disable=C0116 + def data_step(self, dataloader_iter): + return self.config.data_step_fn(dataloader_iter) + + def forward(self, *args, **kwargs): + return self.module(*args, **kwargs) + + def training_step(self, batch, batch_idx=None) -> torch.Tensor: + # In mcore the loss-function is part of the forward-pass (when labels are provided) + return self.forward_step(batch) + + def validation_step(self, batch, batch_idx=None) -> torch.Tensor: + # In mcore the loss-function is part of the forward-pass (when labels are provided) + + return self.forward_step(batch) + + # pylint: disable=C0116 + def forward_step(self, batch) -> torch.Tensor: + # pylint: disable=C0116 + if self.optim.config.bf16: + self.autocast_dtype = torch.bfloat16 + elif self.optim.config.fp16: + self.autocast_dtype = torch.float + else: + self.autocast_dtype = torch.float32 + + if self.image_precached: + latents = batch["latents"].cuda(non_blocking=True) + else: + img = batch["images"].cuda(non_blocking=True) + latents = self.vae.encode(img).to(dtype=self.autocast_dtype) + latents, noise, packed_noisy_model_input, latent_image_ids, guidance_vec, timesteps = ( + self.prepare_image_latent(latents) + ) + if self.text_precached: + prompt_embeds = batch["prompt_embeds"].cuda(non_blocking=True).transpose(0, 1) + pooled_prompt_embeds = batch["pooled_prompt_embeds"].cuda(non_blocking=True) + text_ids = batch["text_ids"].cuda(non_blocking=True) + else: + txt = batch["txt"] + prompt_embeds, pooled_prompt_embeds, text_ids = self.encode_prompt( + txt, device=latents.device, dtype=latents.dtype + ) + with torch.cuda.amp.autocast( + self.autocast_dtype in (torch.half, torch.bfloat16), + dtype=self.autocast_dtype, + ): + noise_pred = self.forward( + img=packed_noisy_model_input, + txt=prompt_embeds, + y=pooled_prompt_embeds, + timesteps=timesteps / 1000, + img_ids=latent_image_ids, + txt_ids=text_ids, + guidance=guidance_vec, + ) + + noise_pred = self._unpack_latents( + noise_pred.transpose(0, 1), + int(latents.shape[2] * self.vae_scale_factor // 2), + int(latents.shape[3] * self.vae_scale_factor // 2), + vae_scale_factor=self.vae_scale_factor, + ).transpose(0, 1) + + target = noise - latents + loss = F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + return loss + + def encode_prompt(self, prompt, device="cuda", dtype=torch.float32): + # pylint: disable=C0116 + prompt_embeds = self.t5(prompt).transpose(0, 1) + _, pooled_prompt_embeds = self.clip(prompt) + text_ids = torch.zeros(prompt_embeds.shape[1], prompt_embeds.shape[0], 3).to(device=device, dtype=dtype) + + return prompt_embeds, pooled_prompt_embeds.to(dtype=dtype), text_ids + + def compute_density_for_timestep_sampling( + self, + weighting_scheme: str, + batch_size: int, + logit_mean: float = 0.0, + logit_std: float = 1.0, + mode_scale: float = 1.29, + ): + """ + Compute the density for sampling the timesteps when doing SD3 training. + + Courtesy: This was contributed by Rafie Walker in https://github.com/huggingface/diffusers/pull/8528. + + SD3 paper reference: https://arxiv.org/abs/2403.03206v1. + """ + if weighting_scheme == "logit_normal": + # See 3.1 in the SD3 paper ($rf/lognorm(0.00,1.00)$). + u = torch.normal(mean=logit_mean, std=logit_std, size=(batch_size,), device="cpu") + u = torch.nn.functional.sigmoid(u) + elif weighting_scheme == "mode": + u = torch.rand(size=(batch_size,), device="cpu") + u = 1 - u - mode_scale * (torch.cos(math.pi * u / 2) ** 2 - 1 + u) + else: + u = torch.rand(size=(batch_size,), device="cpu") + return u + + def prepare_image_latent(self, latents): + # pylint: disable=C0116 + latent_image_ids = self._prepare_latent_image_ids( + latents.shape[0], + latents.shape[2], + latents.shape[3], + latents.device, + latents.dtype, + ) + + noise = torch.randn_like(latents, device=latents.device, dtype=latents.dtype) + batch_size = latents.shape[0] + u = self.compute_density_for_timestep_sampling( + "logit_normal", + batch_size, + ) + indices = (u * self.scheduler.num_train_timesteps).long() + timesteps = self.scheduler.timesteps[indices].to(device=latents.device) + + sigmas = self.scheduler.sigmas.to(device=latents.device, dtype=latents.dtype) + schduler_timesteps = self.scheduler.timesteps.to(device=latents.device) + step_indices = [(schduler_timesteps == t).nonzero().item() for t in timesteps] + timesteps = timesteps.to(dtype=latents.dtype) + sigma = sigmas[step_indices].flatten() + + while len(sigma.shape) < latents.ndim: + sigma = sigma.unsqueeze(-1) + + noisy_model_input = (1.0 - sigma) * latents + sigma * noise + packed_noisy_model_input = self._pack_latents( + noisy_model_input, + batch_size=latents.shape[0], + num_channels_latents=latents.shape[1], + height=latents.shape[2], + width=latents.shape[3], + ) + + if self.config.guidance_embed: + guidance_vec = torch.full( + (noisy_model_input.shape[0],), + self.config.guidance_scale, + device=latents.device, + dtype=latents.dtype, + ) + else: + guidance_vec = None + + return ( + latents.transpose(0, 1), + noise.transpose(0, 1), + packed_noisy_model_input.transpose(0, 1), + latent_image_ids, + guidance_vec, + timesteps, + ) + + def _unpack_latents(self, latents, height, width, vae_scale_factor): + # pylint: disable=C0116 + batch_size, num_patches, channels = latents.shape + + height = height // vae_scale_factor + width = width // vae_scale_factor + + latents = latents.view(batch_size, height, width, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), height * 2, width * 2) + + return latents + + def _prepare_latent_image_ids( + self, batch_size: int, height: int, width: int, device: torch.device, dtype: torch.dtype + ): + # pylint: disable=C0116 + latent_image_ids = torch.zeros(height // 2, width // 2, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height // 2)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width // 2)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids[None, :].repeat(batch_size, 1, 1, 1) + latent_image_ids = latent_image_ids.reshape( + batch_size, latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + + def _pack_latents(self, latents, batch_size, num_channels_latents, height, width): + # pylint: disable=C0116 + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + def set_input_tensor(self, tensor): + # pylint: disable=C0116 + pass + + @property + def training_loss_reduction(self) -> MaskedTokenLossReduction: + # pylint: disable=C0116 + if not self._training_loss_reduction: + self._training_loss_reduction = MaskedTokenLossReduction() + + return self._training_loss_reduction + + @property + def validation_loss_reduction(self) -> MaskedTokenLossReduction: + # pylint: disable=C0116 + # pylint: disable=C0116 + if not self._validation_loss_reduction: + self._validation_loss_reduction = MaskedTokenLossReduction(validation_step=True) + + return self._validation_loss_reduction + + +@io.model_importer(MegatronFluxModel, "hf") +class HFFluxImporter(io.ModelConnector["FluxTransformer2DModel", MegatronFluxModel]): + """ + Convert a HF ckpt into NeMo dist-ckpt compatible format. + """ + + # pylint: disable=C0116 + def init(self) -> MegatronFluxModel: + return MegatronFluxModel(self.config) + + def apply(self, output_path: Path) -> Path: + from diffusers import FluxTransformer2DModel + + source = FluxTransformer2DModel.from_pretrained(str(self), subfolder="transformer") + target = self.init() + trainer = self.nemo_setup(target) + self.convert_state(source, target) + print(f"Converted flux transformer to Nemo, saving to {output_path}") + + self.nemo_save(output_path, trainer) + + print(f"Converted flux transformer saved to {output_path}") + + teardown(trainer, target) + + return output_path + + @property + def config(self) -> FluxConfig: + from diffusers import FluxTransformer2DModel + + source = FluxTransformer2DModel.from_pretrained(str(self), subfolder="transformer") + source_config = source.config + flux_config = FluxConfig( + num_layers=1, # dummy setting + num_joint_layers=source_config.num_layers, + num_single_layers=source_config.num_single_layers, + hidden_size=source_config.num_attention_heads * source_config.attention_head_dim, + num_attention_heads=source_config.num_attention_heads, + activation_func=openai_gelu, + add_qkv_bias=True, + in_channels=source_config.in_channels, + context_dim=source_config.joint_attention_dim, + model_channels=256, + patch_size=source_config.patch_size, + guidance_embed=source_config.guidance_embeds, + vec_in_dim=source_config.pooled_projection_dim, + rotary_interleaved=True, + layernorm_epsilon=1e-06, + hidden_dropout=0, + attention_dropout=0, + use_cpu_initialization=True, + ) + + output = FluxModelParams( + flux_config=flux_config, + vae_config=None, + clip_params=None, + t5_params=None, + scheduler_steps=1000, + device="cuda", + ) + return output + + def convert_state(self, source, target): + # pylint: disable=C0301 + mapping = { + "transformer_blocks.*.norm1.linear.weight": "double_blocks.*.adaln.adaLN_modulation.1.weight", + "transformer_blocks.*.norm1.linear.bias": "double_blocks.*.adaln.adaLN_modulation.1.bias", + "transformer_blocks.*.norm1_context.linear.weight": "double_blocks.*.adaln_context.adaLN_modulation.1.weight", + "transformer_blocks.*.norm1_context.linear.bias": "double_blocks.*.adaln_context.adaLN_modulation.1.bias", + "transformer_blocks.*.attn.norm_q.weight": "double_blocks.*.self_attention.q_layernorm.weight", + "transformer_blocks.*.attn.norm_k.weight": "double_blocks.*.self_attention.k_layernorm.weight", + "transformer_blocks.*.attn.norm_added_q.weight": "double_blocks.*.self_attention.added_q_layernorm.weight", + "transformer_blocks.*.attn.norm_added_k.weight": "double_blocks.*.self_attention.added_k_layernorm.weight", + "transformer_blocks.*.attn.to_out.0.weight": "double_blocks.*.self_attention.linear_proj.weight", + "transformer_blocks.*.attn.to_out.0.bias": "double_blocks.*.self_attention.linear_proj.bias", + "transformer_blocks.*.attn.to_add_out.weight": "double_blocks.*.self_attention.added_linear_proj.weight", + "transformer_blocks.*.attn.to_add_out.bias": "double_blocks.*.self_attention.added_linear_proj.bias", + "transformer_blocks.*.ff.net.0.proj.weight": "double_blocks.*.mlp.linear_fc1.weight", + "transformer_blocks.*.ff.net.0.proj.bias": "double_blocks.*.mlp.linear_fc1.bias", + "transformer_blocks.*.ff.net.2.weight": "double_blocks.*.mlp.linear_fc2.weight", + "transformer_blocks.*.ff.net.2.bias": "double_blocks.*.mlp.linear_fc2.bias", + "transformer_blocks.*.ff_context.net.0.proj.weight": "double_blocks.*.context_mlp.linear_fc1.weight", + "transformer_blocks.*.ff_context.net.0.proj.bias": "double_blocks.*.context_mlp.linear_fc1.bias", + "transformer_blocks.*.ff_context.net.2.weight": "double_blocks.*.context_mlp.linear_fc2.weight", + "transformer_blocks.*.ff_context.net.2.bias": "double_blocks.*.context_mlp.linear_fc2.bias", + "single_transformer_blocks.*.norm.linear.weight": "single_blocks.*.adaln.adaLN_modulation.1.weight", + "single_transformer_blocks.*.norm.linear.bias": "single_blocks.*.adaln.adaLN_modulation.1.bias", + "single_transformer_blocks.*.proj_mlp.weight": "single_blocks.*.mlp.linear_fc1.weight", + "single_transformer_blocks.*.proj_mlp.bias": "single_blocks.*.mlp.linear_fc1.bias", + "single_transformer_blocks.*.attn.norm_q.weight": "single_blocks.*.self_attention.q_layernorm.weight", + "single_transformer_blocks.*.attn.norm_k.weight": "single_blocks.*.self_attention.k_layernorm.weight", + "single_transformer_blocks.*.proj_out.bias": "single_blocks.*.mlp.linear_fc2.bias", + "norm_out.linear.bias": "norm_out.adaLN_modulation.1.bias", + "norm_out.linear.weight": "norm_out.adaLN_modulation.1.weight", + "proj_out.bias": "proj_out.bias", + "proj_out.weight": "proj_out.weight", + "time_text_embed.guidance_embedder.linear_1.bias": "guidance_embedding.in_layer.bias", + "time_text_embed.guidance_embedder.linear_1.weight": "guidance_embedding.in_layer.weight", + "time_text_embed.guidance_embedder.linear_2.bias": "guidance_embedding.out_layer.bias", + "time_text_embed.guidance_embedder.linear_2.weight": "guidance_embedding.out_layer.weight", + "x_embedder.bias": "img_embed.bias", + "x_embedder.weight": "img_embed.weight", + "time_text_embed.timestep_embedder.linear_1.bias": "timestep_embedding.time_embedder.in_layer.bias", + "time_text_embed.timestep_embedder.linear_1.weight": "timestep_embedding.time_embedder.in_layer.weight", + "time_text_embed.timestep_embedder.linear_2.bias": "timestep_embedding.time_embedder.out_layer.bias", + "time_text_embed.timestep_embedder.linear_2.weight": "timestep_embedding.time_embedder.out_layer.weight", + "context_embedder.bias": "txt_embed.bias", + "context_embedder.weight": "txt_embed.weight", + "time_text_embed.text_embedder.linear_1.bias": "vector_embedding.in_layer.bias", + "time_text_embed.text_embedder.linear_1.weight": "vector_embedding.in_layer.weight", + "time_text_embed.text_embedder.linear_2.bias": "vector_embedding.out_layer.bias", + "time_text_embed.text_embedder.linear_2.weight": "vector_embedding.out_layer.weight", + } + return io.apply_transforms( + source, + target, + mapping=mapping, + transforms=[ + import_double_block_qkv, + import_double_block_qkv_bias, + import_added_qkv, + import_added_qkv_bias, + import_single_block_qkv, + import_single_block_qkv_bias, + transform_single_proj_out, + ], + ) + + +@io.state_transform( + source_key=( + "transformer_blocks.*.attn.to_q.weight", + "transformer_blocks.*.attn.to_k.weight", + "transformer_blocks.*.attn.to_v.weight", + ), + target_key=("double_blocks.*.self_attention.linear_qkv.weight"), +) +def import_double_block_qkv(ctx: io.TransformCTX, q, k, v): + transformer_config = ctx.target.config + return _import_qkv(transformer_config, q, k, v) + + +@io.state_transform( + source_key=( + "transformer_blocks.*.attn.to_q.bias", + "transformer_blocks.*.attn.to_k.bias", + "transformer_blocks.*.attn.to_v.bias", + ), + target_key=("double_blocks.*.self_attention.linear_qkv.bias"), +) +def import_double_block_qkv_bias(ctx: io.TransformCTX, qb, kb, vb): + transformer_config = ctx.target.config + return _import_qkv_bias(transformer_config, qb, kb, vb) + + +@io.state_transform( + source_key=( + "transformer_blocks.*.attn.add_q_proj.weight", + "transformer_blocks.*.attn.add_k_proj.weight", + "transformer_blocks.*.attn.add_v_proj.weight", + ), + target_key=("double_blocks.*.self_attention.added_linear_qkv.weight"), +) +def import_added_qkv(ctx: io.TransformCTX, q, k, v): + transformer_config = ctx.target.config + return _import_qkv(transformer_config, q, k, v) + + +@io.state_transform( + source_key=( + "transformer_blocks.*.attn.add_q_proj.bias", + "transformer_blocks.*.attn.add_k_proj.bias", + "transformer_blocks.*.attn.add_v_proj.bias", + ), + target_key=("double_blocks.*.self_attention.added_linear_qkv.bias"), +) +def import_added_qkv_bias(ctx: io.TransformCTX, qb, kb, vb): + transformer_config = ctx.target.config + return _import_qkv_bias(transformer_config, qb, kb, vb) + + +@io.state_transform( + source_key=( + "single_transformer_blocks.*.attn.to_q.weight", + "single_transformer_blocks.*.attn.to_k.weight", + "single_transformer_blocks.*.attn.to_v.weight", + ), + target_key=("single_blocks.*.self_attention.linear_qkv.weight"), +) +def import_single_block_qkv(ctx: io.TransformCTX, q, k, v): + transformer_config = ctx.target.config + return _import_qkv(transformer_config, q, k, v) + + +@io.state_transform( + source_key=( + "single_transformer_blocks.*.attn.to_q.bias", + "single_transformer_blocks.*.attn.to_k.bias", + "single_transformer_blocks.*.attn.to_v.bias", + ), + target_key=("single_blocks.*.self_attention.linear_qkv.bias"), +) +def import_single_block_qkv_bias(ctx: io.TransformCTX, qb, kb, vb): + transformer_config = ctx.target.config + return _import_qkv_bias(transformer_config, qb, kb, vb) + + +@io.state_transform( + source_key=("single_transformer_blocks.*.proj_out.weight"), + target_key=("single_blocks.*.mlp.linear_fc2.weight", "single_blocks.*.self_attention.linear_proj.weight"), +) +def transform_single_proj_out(proj_weight): + linear_fc2 = proj_weight.detach()[:, 3072:].clone() + linear_proj = proj_weight.detach()[:, :3072].clone() + return linear_fc2, linear_proj diff --git a/nemo_vfm/diffusion/models/flux/pipeline.py b/nemo_vfm/diffusion/models/flux/pipeline.py new file mode 100644 index 00000000..43c4a2fc --- /dev/null +++ b/nemo_vfm/diffusion/models/flux/pipeline.py @@ -0,0 +1,1011 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from typing import List, Optional, Union + +import numpy as np +import torch +from PIL import Image +from safetensors.torch import load_file as load_safetensors +from safetensors.torch import save_file as save_safetensors +from torch import nn +from tqdm import tqdm + +from nemo.collections.diffusion.encoders.conditioner import FrozenCLIPEmbedder, FrozenT5Embedder +from nemo.collections.diffusion.models.flux.model import Flux +from nemo.collections.diffusion.models.flux_controlnet.model import FluxControlNet, FluxControlNetConfig +from nemo.collections.diffusion.sampler.flow_matching.flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler +from nemo.collections.diffusion.utils.flux_ckpt_converter import flux_transformer_converter +from nemo.collections.diffusion.utils.flux_pipeline_utils import FluxModelParams +from nemo.collections.diffusion.vae.autoencoder import AutoEncoder +from nemo.utils import logging + + +class FluxInferencePipeline(nn.Module): + """ + A pipeline for performing image generation with flux. + + Args: + params (FluxModelParams, optional): + Configuration parameters for the model pipeline, including device settings and model configurations. + flux (Flux, optional): + A pre-initialized Flux model used for the transformation process. + If None, a new Flux model is created using the configuration in `params`. + vae (AutoEncoder, optional): + A pre-initialized VAE (Variational Autoencoder) model. + If None, a new VAE model is created using the configuration in `params.vae_config`. + t5 (FrozenT5Embedder, optional): + A pre-initialized FrozenT5Embedder model. + If None, a new T5 model is created using the configuration in `params.t5_params`. + clip (FrozenCLIPEmbedder, optional): + A pre-initialized FrozenCLIPEmbedder model. + If None, a new CLIP model is created using the configuration in `params.clip_params`. + scheduler_steps (int, optional): + The number of scheduler steps to use for inference. Default is 1000. + + Attributes: + device (torch.device): The device (CPU or GPU) where the models will be placed. + vae (AutoEncoder): The VAE model used for image reconstruction or generation. + clip_encoder (FrozenCLIPEmbedder): The CLIP encoder for processing image-text inputs. + t5_encoder (FrozenT5Embedder): The T5 encoder for processing text inputs. + transformer (Flux): The Flux model used for image-text joint processing. + vae_scale_factor (float): A scale factor for the VAE, based on the number of channels in the VAE. + scheduler (FlowMatchEulerDiscreteScheduler): Scheduler used for controlling the flow of inference steps. + params (FluxModelParams): Configuration parameters used for model setup. + + Methods: + load_from_pretrained: + Loads model weights from a checkpoint. + encoder_prompt: + Encodes text prompts and retrieves embeddings. + _prepare_latent_image_ids: + Prepares latent image ids for the generation process. + _pack_latents: + Packs latents into the desired format for input to the model. + _unpack_latents: + Unpacks latents from the model into image format. + _calculate_shift: + Calculates the shift parameter used for controlling sequence lengths in the model. + prepare_latents: + Prepares the latent tensors and latent image ids for generation. + _generate_rand_latents: + Generates random latents using a specified generator. + numpy_to_pil: + Converts a numpy array or a batch of images to PIL images. + torch_to_numpy: + Converts a tensor of images to a numpy array. + denormalize: + Denormalizes the image to the range [0, 1]. + __call__: + Runs the entire image generation process based on the input prompt, including encoding, + latent preparation, inference, and output generation. + + Example: + pipeline = FluxInferencePipeline(params) + images = pipeline( + prompt=["A beautiful sunset over a mountain range"], + height=512, + width=512, + num_inference_steps=50, + guidance_scale=7.5 + ) + """ + + def __init__( + self, + params: FluxModelParams = None, + flux: Optional[Flux] = None, + vae: Optional[AutoEncoder] = None, + t5: Optional[FrozenT5Embedder] = None, + clip: Optional[FrozenCLIPEmbedder] = None, + scheduler_steps: int = 1000, + ): + """ + Initializes the FluxInferencePipeline with the provided models and configurations. + + Args: + params (FluxModelParams, optional): + Configuration parameters for the model pipeline, including device settings and model configurations. + flux (Flux, optional): + A pre-initialized Flux model used for the transformation process. + If None, a new Flux model is created using the configuration in `params`. + vae (AutoEncoder, optional): + A pre-initialized VAE (Variational Autoencoder) model. + If None, a new VAE model is created using the configuration in `params.vae_config`. + t5 (FrozenT5Embedder, optional): + A pre-initialized FrozenT5Embedder model. + If None, a new T5 model is created using the configuration in `params.t5_params`. + clip (FrozenCLIPEmbedder, optional): + A pre-initialized FrozenCLIPEmbedder model. + If None, a new CLIP model is created using the configuration in `params.clip_params`. + scheduler_steps (int, optional): The number of scheduler steps to use for inference. Default is 1000. + """ + super().__init__() + self.device = params.device + params.clip_params.device = self.device + params.t5_params.device = self.device + + self.vae = AutoEncoder(params.vae_config).to(self.device).eval() if vae is None else vae + self.clip_encoder = ( + FrozenCLIPEmbedder( + version=params.clip_params.version, + max_length=params.clip_params.max_length, + always_return_pooled=params.clip_params.always_return_pooled, + device=params.clip_params.device, + ) + if clip is None + else clip + ) + self.t5_encoder = ( + FrozenT5Embedder( + params.t5_params.version, max_length=params.t5_params.max_length, device=params.t5_params.device + ) + if t5 is None + else t5 + ) + self.transformer = Flux(params.flux_config).to(self.device).eval() if flux is None else flux + self.vae_scale_factor = 2 ** (len(self.vae.params.ch_mult)) + self.scheduler = FlowMatchEulerDiscreteScheduler(num_train_timesteps=scheduler_steps) + self.params = params + + def load_from_pretrained(self, ckpt_path, do_convert_from_hf=True, save_converted_model_to=None): + """ + Loads the model's weights from a checkpoint. If HF ckpt is provided, it will be converted to NeMo + format and save it to local folder. + + Args: + ckpt_path (str): + Path to the checkpoint file. + do_convert_from_hf (bool, optional): + Whether to convert the checkpoint from Hugging Face format before loading. Default is True. + save_converted_model_to (str, optional): + Path to save the converted checkpoint if `do_convert_from_hf` is True. Default is None. + + Logs: + The function logs information about missing or unexpected keys during checkpoint loading. + """ + if do_convert_from_hf: + ckpt = flux_transformer_converter(ckpt_path, self.transformer.config) + if save_converted_model_to is not None: + save_path = os.path.join(save_converted_model_to, "nemo_flux_transformer.safetensors") + save_safetensors(ckpt, save_path) + logging.info(f"saving converted transformer checkpoint to {save_path}") + else: + ckpt = load_safetensors(ckpt_path) + missing, unexpected = self.transformer.load_state_dict(ckpt, strict=False) + missing = [k for k in missing if not k.endswith("_extra_state")] + # These keys are mcore specific and should not affect the model performance + if len(missing) > 0: + logging.info( + f"The following keys are missing during checkpoint loading, " + f"please check the ckpt provided or the image quality may be compromised.\n {missing}" + ) + logging.info(f"Found unexepected keys: \n {unexpected}") + + def encoder_prompt( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = "cuda", + dtype: Optional[torch.dtype] = torch.float, + ): + """ + Encodes a text prompt (or a batch of prompts) into embeddings using both T5 and CLIP models. + + Args: + prompt (Union[str, List[str]]): + The text prompt(s) to be encoded. Can be a string or a list of strings. + num_images_per_prompt (int, optional): + The number of images to generate per prompt. Default is 1. + prompt_embeds (torch.FloatTensor, optional): + Precomputed prompt embeddings, if available. Default is None. + pooled_prompt_embeds (torch.FloatTensor, optional): + Precomputed pooled prompt embeddings, if available. Default is None. + max_sequence_length (int, optional): + The maximum sequence length for the text model. Default is 512. + device (torch.device, optional): + The device (CPU or CUDA) on which the models are placed. Default is 'cuda'. + dtype (torch.dtype, optional): + The data type for tensor operations. Default is `torch.float`. + + Returns: + Tuple[torch.FloatTensor, torch.FloatTensor, torch.FloatTensor]: + - The prompt embeddings. + - The pooled prompt embeddings. + - The text IDs for the prompt. + + Raises: + ValueError: If neither `prompt` nor `prompt_embeds` are provided. + """ + if prompt is not None: + batch_size = len(prompt) + elif prompt_embeds is not None: + batch_size = prompt_embeds.shape[0] + else: + raise ValueError("Either prompt or prompt_embeds must be provided.") + if device == "cuda" and self.t5_encoder.device != device: + self.t5_encoder.to(device) + if prompt_embeds is None: + prompt_embeds = self.t5_encoder(prompt, max_sequence_length=max_sequence_length) + seq_len = prompt_embeds.shape[1] + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1).to(dtype=dtype) + + if device == "cuda" and self.clip_encoder.device != device: + self.clip_encoder.to(device) + if pooled_prompt_embeds is None: + _, pooled_prompt_embeds = self.clip_encoder(prompt) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt, 1) + pooled_prompt_embeds = pooled_prompt_embeds.view(batch_size * num_images_per_prompt, -1).to(dtype=dtype) + + dtype = dtype if dtype is not None else self.t5_encoder.dtype + text_ids = torch.zeros(batch_size, prompt_embeds.shape[1], 3).to(device=device, dtype=dtype) + text_ids = text_ids.repeat(num_images_per_prompt, 1, 1) + + return prompt_embeds.transpose(0, 1), pooled_prompt_embeds, text_ids + + @staticmethod + def _prepare_latent_image_ids(batch_size: int, height: int, width: int, device: torch.device, dtype: torch.dtype): + """ + Prepares latent image IDs for input into the model. These IDs represent the image grid. + + Args: + batch_size (int): The number of samples in the batch. + height (int): The height of the image. + width (int): The width of the image. + device (torch.device): The device to place the tensor. + dtype (torch.dtype): The data type for the tensor. + + Returns: + torch.FloatTensor: A tensor representing the latent image IDs. + """ + latent_image_ids = torch.zeros(height // 2, width // 2, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height // 2)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width // 2)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids[None, :].repeat(batch_size, 1, 1, 1) + latent_image_ids = latent_image_ids.reshape( + batch_size, latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + + @staticmethod + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + """ + Packs latents into desired shape, e.g. (B, C, H, W) --> (B, (H//2)*(W//2), C * 4). + + Args: + latents (torch.Tensor): The latents to be packed. + batch_size (int): The number of samples in the batch. + num_channels_latents (int): The number of channels in the latents. + height (int): The height of the image. + width (int): The width of the image. + + Returns: + torch.Tensor: The packed latents. + """ + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + def _unpack_latents(latents, height, width, vae_scale_factor): + """ + Unpacks the latents from the model output into an image format suitable for further processing. + + The method reshapes and permutes the latents, adjusting their dimensions according to the + specified `vae_scale_factor` to match the expected resolution of the image. + + Args: + latents (torch.Tensor): The latents output from the model, typically in a compact, compressed format. + height (int): The original height of the image before scaling, used to adjust the latent dimensions. + width (int): The original width of the image before scaling, used to adjust the latent dimensions. + vae_scale_factor (int): A scale factor used to adjust the resolution of the image when unpacking. + This factor istypically the inverse of the VAE downsampling factor. + + Returns: + torch.Tensor: The unpacked latents reshaped to match the expected dimensions for image reconstruction. + The output tensor will have shape `(batch_size, channels, height * 2, width * 2)`. + + Notes: + - This function is intended to convert latents back into a format + that can be decoded into images by the VAE. + """ + batch_size, num_patches, channels = latents.shape + + height = height // vae_scale_factor + width = width // vae_scale_factor + + latents = latents.view(batch_size, height, width, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), height * 2, width * 2) + + return latents + + @staticmethod + def _calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.16, + ): + # pylint: disable=C0116 + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + """ + Prepares and optionally generates image latents for use in the image generation pipeline. + + This method can either use the provided latents (if already available) or generate new random latents + using a random generator. The generated latents are then packed and prepared for the model to process. + + Args: + batch_size (int): The number of samples in the batch. + num_channels_latents (int): The number of channels in the latents (e.g., depth of the latent tensor). + height (int): The height of the image to be generated (before scaling). + width (int): The width of the image to be generated (before scaling). + dtype (torch.dtype): The data type to use for the latents (e.g., `torch.float32`). + device (torch.device): The device on which the latents will reside (e.g., 'cuda'). + generator (Union[torch.Generator, List[torch.Generator]]): A random number generator or a list of generators + for generating random latents. If a list is provided, its length must match the batch size. + latents (Optional[torch.FloatTensor]): An optional pre-existing latent tensor. If provided, it is used + instead of generating new latents. + + Returns: + tuple: A tuple containing: + - latents (torch.Tensor): + The prepared latents, with shape `(batch_size, num_channels_latents, height, width)`. + - latent_image_ids (torch.Tensor): + A tensor containing latent image IDs for each batch sample, used for indexing + in the model. + + Raises: + ValueError: If a list of generators is provided but its length does not match the batch size. + + """ + height = 2 * int(height) // self.vae_scale_factor + width = 2 * int(width) // self.vae_scale_factor + + shape = (batch_size, num_channels_latents, height, width) + + if latents is not None: + latent_image_ids = self._prepare_latent_image_ids(batch_size, height, width, device, dtype) + return latents.to(device=device, dtype=dtype), latent_image_ids + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + latents = FluxInferencePipeline._generate_rand_latents(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + + latent_image_ids = self._prepare_latent_image_ids(batch_size, height, width, device, dtype) + + return latents.transpose(0, 1), latent_image_ids + + @staticmethod + def _generate_rand_latents( + shape, + generator, + device, + dtype, + ): + """ + Create random latents using a random generator or a list of generators. + """ + if isinstance(generator, list): + shape = (1,) + shape[1:] + latents = [ + torch.randn(shape, generator=generator[i], device=device, dtype=dtype, layout=layout) + for i in range(batch_size) + ] + latents = torch.cat(latents, dim=0).to(device=device) + else: + latents = torch.randn(shape, generator=generator, device=device, dtype=dtype) + + return latents + + @staticmethod + def numpy_to_pil(images): + """ + Convert a numpy image or a batch of images to a PIL image. + """ + if images.ndim == 3: + images = images[None, ...] + images = (images * 255).round().astype("uint8") + pil_images = [Image.fromarray(image) for image in images] + + return pil_images + + @staticmethod + def torch_to_numpy(images): + """ + Convert a torch image or a batch of images to a numpy image. + """ + numpy_images = images.float().cpu().permute(0, 2, 3, 1).numpy() + return numpy_images + + @staticmethod + def denormalize(image): + # pylint: disable=C0116 + return (image / 2 + 0.5).clamp(0, 1) + + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = 512, + width: Optional[int] = 512, + num_inference_steps: int = 28, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 7.0, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + max_sequence_length: int = 512, + device: torch.device = "cuda", + dtype: torch.dtype = torch.float32, + save_to_disk: bool = True, + offload: bool = False, + ): + """ + Generates images based on a given text prompt and various model parameters. Optionally saves the images to disk. + + This method orchestrates the process of generating images by embedding the prompt, preparing the latent vectors, + iterating through timesteps in the diffusion process, and then decoding the latent representation back into + an image. It supports both the generation of latent representations or final images in a desired output format + (e.g., PIL image). The images are optionally saved to disk with a unique filename based on the prompt. + + Args: + prompt (Union[str, List[str]]): + A text prompt or a list of text prompts to guide image generation. Each prompt + generates one or more images based on the `num_images_per_prompt`. + height (Optional[int]): + The height of the output image. Default is 512. + width (Optional[int]): + The width of the output image. Default is 512. + num_inference_steps (int): + The number of steps for the diffusion process. Default is 28. + timesteps (Optional[List[int]]): + A list of specific timesteps for the diffusion process. If not provided, + they are automatically calculated. + guidance_scale (float): + The scale of the guidance signal, typically used to control the strength of prompt conditioning. + num_images_per_prompt (Optional[int]): + The number of images to generate per prompt. Default is 1. + generator (Optional[Union[torch.Generator, List[torch.Generator]]]): + A random number generator or a list of generators + for generating latents. If a list is provided, it should match the batch size. + latents (Optional[torch.FloatTensor]): + Pre-existing latents to use instead of generating new ones. + prompt_embeds (Optional[torch.FloatTensor]): + Optionally pre-computed prompt embeddings to skip the prompt encoding step. + pooled_prompt_embeds (Optional[torch.FloatTensor]): + Optionally pre-computed pooled prompt embeddings. + output_type (Optional[str]): + The format of the output. Can be "latent" or "pil" (PIL image). Default is "pil". + max_sequence_length (int): + The maximum sequence length for tokenizing the prompt. Default is 512. + device (torch.device): + The device on which the computation should take place (e.g., 'cuda'). Default is 'cuda'. + dtype (torch.dtype): + The data type of the latents and model weights. Default is `torch.float32`. + save_to_disk (bool): + Whether or not to save the generated images to disk. Default is True. + offload (bool): + Whether or not to offload model components to CPU to free up GPU memory during the process. + Default is False. + + Returns: + Union[List[Image.Image], torch.Tensor]: + The generated images or latents, depending on the `output_type` argument. + If `output_type` is "pil", a list of PIL images is returned. If "latent", the latents are returned. + + Raises: + ValueError: If neither a `prompt` nor `prompt_embeds` is provided. + + Notes: + - The model expects a device of 'cuda'. + The method will raise an assertion error if a different device is provided. + - The method handles both prompt-based and pre-embedded prompt input, + providing flexibility for different usage scenarios. + - If `save_to_disk` is enabled, images will be saved with a filename derived from the prompt text. + """ + assert device == "cuda", "Transformer blocks in Mcore must run on cuda devices" + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + prompt = [prompt] + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + elif prompt_embeds is not None and isinstance(prompt_embeds, torch.FloatTensor): + batch_size = prompt_embeds.shape[0] + else: + raise ValueError("Either prompt or prompt_embeds must be provided.") + + # get text prompt embeddings + prompt_embeds, pooled_prompt_embeds, text_ids = self.encoder_prompt( + prompt=prompt, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + if offload: + self.t5_encoder.to("cpu") + self.clip_encoder.to("cpu") + torch.cuda.empty_cache() + + # prepare image latents + num_channels_latents = self.transformer.in_channels // 4 + latents, latent_image_ids = self.prepare_latents( + batch_size * num_images_per_prompt, num_channels_latents, height, width, dtype, device, generator, latents + ) + # prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) + image_seq_len = latents.shape[0] + + mu = FluxInferencePipeline._calculate_shift( + image_seq_len, + self.scheduler.base_image_seq_len, + self.scheduler.max_image_seq_len, + self.scheduler.base_shift, + self.scheduler.max_shift, + ) + + self.scheduler.set_timesteps(sigmas=sigmas, device=device, mu=mu) + timesteps = self.scheduler.timesteps + + if device == "cuda" and device != self.device: + self.transformer.to(device) + with torch.no_grad(): + for i, t in tqdm(enumerate(timesteps)): + timestep = t.expand(latents.shape[1]).to(device=latents.device, dtype=latents.dtype) + if self.transformer.guidance_embed: + guidance = torch.tensor([guidance_scale], device=device).expand(latents.shape[1]) + else: + guidance = None + with torch.autocast(device_type="cuda", dtype=latents.dtype): + pred = self.transformer( + img=latents, + txt=prompt_embeds, + y=pooled_prompt_embeds, + timesteps=timestep / 1000, + img_ids=latent_image_ids, + txt_ids=text_ids, + guidance=guidance, + ) + latents = self.scheduler.step(pred, t, latents)[0] + if offload: + self.transformer.to("cpu") + torch.cuda.empty_cache() + + if output_type == "latent": + return latents.transpose(0, 1) + elif output_type == "pil": + latents = self._unpack_latents(latents.transpose(0, 1), height, width, self.vae_scale_factor) + if device == "cuda" and device != self.device: + self.vae.to(device) + with torch.autocast(device_type="cuda", dtype=latents.dtype): + image = self.vae.decode(latents) + if offload: + self.vae.to("cpu") + torch.cuda.empty_cache() + image = FluxInferencePipeline.denormalize(image) + image = FluxInferencePipeline.torch_to_numpy(image) + image = FluxInferencePipeline.numpy_to_pil(image) + if save_to_disk: + print("Saving to disk") + assert len(image) == int(len(prompt) * num_images_per_prompt) + prompt = [p[:40] + f"_{idx}" for p in prompt for idx in range(num_images_per_prompt)] + for file_name, image in zip(prompt, image): + image.save(f"{file_name}.png") + + return image + + +class FluxControlNetInferencePipeline(FluxInferencePipeline): + def __init__( + self, + params: Optional[FluxModelParams] = None, + contorlnet_config: Optional[FluxControlNetConfig] = None, + flux: Flux = None, + vae: AutoEncoder = None, + t5: FrozenT5Embedder = None, + clip: FrozenCLIPEmbedder = None, + scheduler_steps: int = 1000, + flux_controlnet: FluxControlNet = None, + ): + """ + Flux Contronlnet inference pipeline initializes controlnet component in addition to a normal flux pipeline. + """ + super().__init__( + params, + flux, + vae, + t5, + clip, + scheduler_steps, + ) + self.flux_controlnet = FluxControlNet(contorlnet_config) if flux_controlnet is None else flux_controlnet + + def load_from_pretrained( + self, flux_ckpt_path, controlnet_ckpt_path, do_convert_from_hf=True, save_converted_model_to=None + ): + """ + Converts both flux base model and flux controlnet ckpt into NeMo format. + """ + if do_convert_from_hf: + flux_ckpt = flux_transformer_converter(flux_ckpt_path, self.transformer.config) + flux_controlnet_ckpt = flux_transformer_converter(controlnet_ckpt_path, self.flux_controlnet.config) + + if save_converted_model_to is not None: + save_path = os.path.join(save_converted_model_to, "nemo_flux_transformer.safetensors") + save_safetensors(flux_ckpt, save_path) + logging.info(f"saving converted transformer checkpoint to {save_path}") + save_path = os.path.join(save_converted_model_to, "nemo_flux_controlnet_transformer.safetensors") + save_safetensors(flux_controlnet_ckpt, save_path) + logging.info(f"saving converted transformer checkpoint to {save_path}") + else: + flux_ckpt = load_safetensors(flux_ckpt_path) + flux_controlnet_ckpt = load_safetensors(controlnet_ckpt_path) + missing, unexpected = self.transformer.load_state_dict(flux_ckpt, strict=False) + missing = [k for k in missing if not k.endswith("_extra_state")] + # These keys are mcore specific and should not affect the model performance + if len(missing) > 0: + logging.info( + f"The following keys are missing during flux checkpoint loading, " + f"please check the ckpt provided or the image quality may be compromised.\n {missing}" + ) + logging.info(f"Found unexepected keys: \n {unexpected}") + + missing, unexpected = self.flux_controlnet.load_state_dict(flux_controlnet_ckpt, strict=False) + missing = [k for k in missing if not k.endswith("_extra_state")] + # These keys are mcore specific and should not affect the model performance + if len(missing) > 0: + logging.info( + f"The following keys are missing during controlnet checkpoint loading, " + f"please check the ckpt provided or the image quality may be compromised.\n {missing}" + ) + logging.info(f"Found unexepected keys: \n {unexpected}") + + def pil_to_numpy(self, images): + """ + PIL image to numpy array + """ + if not isinstance(images, list): + images = [images] + images = [np.array(image).astype(np.float32) / 255.0 for image in images] + images = np.stack(images, axis=0) + + return images + + def numpy_to_pt(self, images: np.ndarray) -> torch.Tensor: + """ + Convert numpy image into torch tensors + """ + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + def prepare_image( + self, + images, + height, + width, + batch_size, + num_images_per_prompt, + device, + dtype, + ): + """ + Preprocess image into torch tensor, also duplicate by batch size. + """ + if isinstance(images, torch.Tensor): + pass + else: + orig_height, orig_width = images[0].height, images[0].width + if height != orig_height or width != orig_width: + images = [image.resize((width, height), resample=3) for image in images] + + images = self.pil_to_numpy(images) + images = self.numpy_to_pt(images) + image_batch_size = images.shape[0] + if image_batch_size == 1: + repeat_by = batch_size + else: + repeat_by = num_images_per_prompt + images = images.repeat_interleave(repeat_by, dim=0) + + images = images.to(device=device, dtype=dtype) + + return images + + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = 512, + width: Optional[int] = 512, + num_inference_steps: int = 28, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 7.0, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + max_sequence_length: int = 512, + device: torch.device = "cuda", + dtype: torch.dtype = torch.float32, + save_to_disk: bool = True, + offload: bool = False, + control_guidance_start: float = 0.0, + control_guidance_end: float = 1.0, + control_image: Union[Image.Image, torch.FloatTensor] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + ): + """ + Generates images based on a given text prompt and optionally incorporates control images and ControlNet for + guidance. + + This method generates images by embedding the prompt, preparing the latent vectors, iterating through timesteps + in the diffusion process, and then decoding the latent representation back into an image. The method supports + control images through ControlNet, where the `control_image` is used to condition the image generation. + It also allows you to specify custom guidance scales and other parameters. Generated images can be saved to disk if requested. + + Args: + prompt (Union[str, List[str]]): + A text prompt or a list of text prompts to guide image generation. Each prompt generates one or more + images based on the `num_images_per_prompt`. + height (Optional[int]): + The height of the output image. Default is 512. + width (Optional[int]): + The width of the output image. Default is 512. + num_inference_steps (int): + The number of steps for the diffusion process. Default is 28. + timesteps (Optional[List[int]]): + A list of specific timesteps for the diffusion process. If not provided, they are automatically + calculated. + guidance_scale (float): + The scale of the guidance signal, typically used to control the strength of prompt conditioning. + num_images_per_prompt (Optional[int]): + The number of images to generate per prompt. Default is 1. + generator (Optional[Union[torch.Generator, List[torch.Generator]]]): + A random number generator or a list of generators for generating latents. If a list is provided, + it should match the batch size. + latents (Optional[torch.FloatTensor]): + Pre-existing latents to use instead of generating new ones. + prompt_embeds (Optional[torch.FloatTensor]): + Optionally pre-computed prompt embeddings to skip the prompt encoding step. + pooled_prompt_embeds (Optional[torch.FloatTensor]): + Optionally pre-computed pooled prompt embeddings. + output_type (Optional[str]): + The format of the output. Can be "latent" or "pil" (PIL image). Default is "pil". + max_sequence_length (int): + The maximum sequence length for tokenizing the prompt. Default is 512. + device (torch.device): + The device on which the computation should take place (e.g., 'cuda'). Default is 'cuda'. + dtype (torch.dtype): + The data type of the latents and model weights. Default is `torch.float32`. + save_to_disk (bool): + Whether or not to save the generated images to disk. Default is True. + offload (bool): + Whether or not to offload model components to CPU to free up GPU memory during the process. Default is False. + control_guidance_start (float): + The start point for control guidance to apply during the diffusion process. + control_guidance_end (float): + The end point for control guidance to apply during the diffusion process. + control_image (Union[Image.Image, torch.FloatTensor]): + The image used for conditioning the generation process via ControlNet. + controlnet_conditioning_scale (Union[float, List[float]]): + Scaling factors to control the impact of the control image in the generation process. + Can be a single value or a list for multiple images. + + Returns: + Union[List[Image.Image], torch.Tensor]: + The generated images or latents, depending on the `output_type` argument. + If `output_type` is "pil", a list of PIL images is returned. If "latent", the latents are returned. + + Raises: + ValueError: If neither a `prompt` nor `prompt_embeds` is provided. + + Notes: + - The model expects a device of 'cuda'. + The method will raise an assertion error if a different device is provided. + - The method supports conditional image generation using ControlNet, where a `control_image` can guide the + generation process. + - If `save_to_disk` is enabled, images will be saved with a filename derived from the prompt text. + """ + assert device == "cuda", "Transformer blocks in Mcore must run on cuda devices" + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + prompt = [prompt] + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + elif prompt_embeds is not None and isinstance(prompt_embeds, torch.FloatTensor): + batch_size = prompt_embeds.shape[0] + else: + raise ValueError("Either prompt or prompt_embeds must be provided.") + + # get text prompt embeddings + prompt_embeds, pooled_prompt_embeds, text_ids = self.encoder_prompt( + prompt=prompt, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + if offload: + self.t5_encoder.to("cpu") + self.clip_encoder.to("cpu") + torch.cuda.empty_cache() + + # prepare image latents + num_channels_latents = self.transformer.in_channels // 4 + latents, latent_image_ids = self.prepare_latents( + batch_size * num_images_per_prompt, num_channels_latents, height, width, dtype, device, generator, latents + ) + + # prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) + image_seq_len = latents.shape[0] + + mu = FluxInferencePipeline._calculate_shift( + image_seq_len, + self.scheduler.base_image_seq_len, + self.scheduler.max_image_seq_len, + self.scheduler.base_shift, + self.scheduler.max_shift, + ) + + self.scheduler.set_timesteps(sigmas=sigmas, device=device, mu=mu) + timesteps = self.scheduler.timesteps + + control_image = self.prepare_image( + images=control_image, + height=height, + width=width, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=torch.float32, + ) + + height, width = control_image.shape[-2:] + if self.flux_controlnet.input_hint_block is None: + if device == "cuda" and self.device != device: + self.vae.to(device) + with torch.no_grad(): + control_image = self.vae.encode(control_image).to(dtype=dtype) + + height_control_image, width_control_image = control_image.shape[2:] + control_image = self._pack_latents( + control_image, + batch_size * num_images_per_prompt, + num_channels_latents, + height_control_image, + width_control_image, + ).transpose(0, 1) + + controlnet_keep = [] + for i in range(len(timesteps)): + controlnet_keep.append( + 1.0 + - float(i / len(timesteps) < control_guidance_start or (i + 1) / len(timesteps) > control_guidance_end) + ) + if device == "cuda" and device != self.device: + self.transformer.to(device) + self.flux_controlnet.to(device) + with torch.no_grad(): + for i, t in tqdm(enumerate(timesteps)): + timestep = t.expand(latents.shape[1]).to(device=latents.device, dtype=latents.dtype) + if self.transformer.guidance_embed: + guidance = torch.tensor([guidance_scale], device=device).expand(latents.shape[1]) + else: + guidance = None + + conditioning_scale = controlnet_keep[i] * controlnet_conditioning_scale + + with torch.autocast(device_type="cuda", dtype=latents.dtype): + controlnet_double_block_samples, controlnet_single_block_samples = self.flux_controlnet( + img=latents, + controlnet_cond=control_image, + txt=prompt_embeds, + y=pooled_prompt_embeds, + timesteps=timestep / 1000, + img_ids=latent_image_ids, + txt_ids=text_ids, + guidance=guidance, + conditioning_scale=conditioning_scale, + ) + pred = self.transformer( + img=latents, + txt=prompt_embeds, + y=pooled_prompt_embeds, + timesteps=timestep / 1000, + img_ids=latent_image_ids, + txt_ids=text_ids, + guidance=guidance, + controlnet_double_block_samples=controlnet_double_block_samples, + controlnet_single_block_samples=controlnet_single_block_samples, + ) + latents = self.scheduler.step(pred, t, latents)[0] + if offload: + self.transformer.to("cpu") + torch.cuda.empty_cache() + + if output_type == "latent": + return latents.transpose(0, 1) + elif output_type == "pil": + latents = self._unpack_latents(latents.transpose(0, 1), height, width, self.vae_scale_factor) + if device == "cuda" and device != self.device: + self.vae.to(device) + with torch.autocast(device_type="cuda", dtype=latents.dtype): + image = self.vae.decode(latents) + if offload: + self.vae.to("cpu") + torch.cuda.empty_cache() + image = FluxInferencePipeline.denormalize(image) + image = FluxInferencePipeline.torch_to_numpy(image) + image = FluxInferencePipeline.numpy_to_pil(image) + if save_to_disk: + print("Saving to disk") + assert len(image) == int(len(prompt) * num_images_per_prompt) + prompt = [p[:40] + f"_{idx}" for p in prompt for idx in range(num_images_per_prompt)] + for file_name, image in zip(prompt, image): + image.save(f"{file_name}.png") + + return image + + +# flake8: noqa diff --git a/nemo_vfm/diffusion/models/flux_controlnet/__init__.py b/nemo_vfm/diffusion/models/flux_controlnet/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/models/flux_controlnet/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/models/flux_controlnet/layers.py b/nemo_vfm/diffusion/models/flux_controlnet/layers.py new file mode 100644 index 00000000..5c800c76 --- /dev/null +++ b/nemo_vfm/diffusion/models/flux_controlnet/layers.py @@ -0,0 +1,88 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Tuple + +import torch.nn as nn +from torch.nn import functional as F + + +class ControlNetConditioningEmbedding(nn.Module): + """ + Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN + [11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized + training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the + convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides + (activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full + model) to encode image-space conditions ... into feature maps ..." + """ + + def __init__( + self, + conditioning_embedding_channels: int, + conditioning_channels: int = 3, + block_out_channels: Tuple[int, ...] = (16, 32, 96, 256), + ): + """ + Initializes the model with convolutional layers for processing conditioning inputs. + + Args: + conditioning_embedding_channels (int): + Number of output channels for the conditioning embedding. + conditioning_channels (int): + Number of input channels for the conditioning data. Default is 3. + block_out_channels (Tuple[int, ...]): + Tuple specifying the output channels for each block. Default is (16, 32, 96, 256). + """ + super().__init__() + + self.conv_in = nn.Conv2d(conditioning_channels, block_out_channels[0], kernel_size=3, padding=1) + + self.blocks = nn.ModuleList([]) + + for i in range(len(block_out_channels) - 1): + channel_in = block_out_channels[i] + channel_out = block_out_channels[i + 1] + self.blocks.append(nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1)) + self.blocks.append(nn.Conv2d(channel_in, channel_out, kernel_size=3, padding=1, stride=2)) + + self.conv_out = zero_module( + nn.Conv2d(block_out_channels[-1], conditioning_embedding_channels, kernel_size=3, padding=1) + ) + + def forward(self, conditioning): + """ + Passes the conditioning input through the model to produce an embedding. + + Args: + conditioning (torch.Tensor): Input tensor representing conditioning data. + + Returns: + torch.Tensor: The resulting embedding tensor after processing through the network. + """ + embedding = self.conv_in(conditioning) + embedding = F.silu(embedding) + + for block in self.blocks: + embedding = block(embedding) + embedding = F.silu(embedding) + + embedding = self.conv_out(embedding) + + return embedding + + +# flake8: noqa diff --git a/nemo_vfm/diffusion/models/flux_controlnet/model.py b/nemo_vfm/diffusion/models/flux_controlnet/model.py new file mode 100644 index 00000000..277bc301 --- /dev/null +++ b/nemo_vfm/diffusion/models/flux_controlnet/model.py @@ -0,0 +1,578 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from dataclasses import dataclass +from typing import Callable + +import torch +import torch.nn as nn +from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.tensor_parallel.layers import ColumnParallelLinear +from megatron.core.transformer.transformer_config import TransformerConfig +from nemo.collections.diffusion.models.dit.dit_layer_spec import ( + FluxSingleTransformerBlock, + MMDiTLayer, + get_flux_double_transformer_engine_spec, + get_flux_single_transformer_engine_spec, +) +from nemo.collections.diffusion.models.flux.layers import EmbedND, MLPEmbedder, TimeStepEmbedder +from nemo.collections.diffusion.models.flux.model import FluxConfig, FluxModelParams, MegatronFluxModel +from nemo.collections.diffusion.models.flux_controlnet.layers import ControlNetConditioningEmbedding +from nemo.lightning import io +from nemo.utils import logging +from torch.nn import functional as F + + +def zero_module(module): + """ + Initializes all parameters of the given module to zero. + + Args: + module (nn.Module): The module whose parameters will be initialized to zero. + + Returns: + nn.Module: The same module with zero-initialized parameters. + """ + for p in module.parameters(): + nn.init.zeros_(p) + return module + + +def flux_controlnet_data_step(dataloader_iter): + """ + Processes a single step of data from a dataloader iterator for the Flux ControlNet. + + Args: + dataloader_iter (Iterator): An iterator over the dataloader that provides batches of data. + + Returns: + dict: A processed batch dictionary with an added 'loss_mask' key. + """ + batch = next(dataloader_iter) + if isinstance(batch, tuple) and len(batch) == 3: + _batch = batch[0] + else: + _batch = batch + + _batch["loss_mask"] = torch.Tensor([1.0]).cuda(non_blocking=True) + return _batch + + +@dataclass +class FluxControlNetConfig(TransformerConfig, io.IOMixin): + """ + Flux config inherits from TransformerConfig class. + """ + + num_layers: int = 1 # dummy setting + patch_size: int = 1 + in_channels: int = 64 + num_joint_layers: int = 4 + num_single_layers: int = 10 + hidden_size: int = 3072 + num_attention_heads: int = 24 + vec_in_dim: int = 768 + context_dim: int = 4096 + guidance_embed: bool = True + num_mode: int = None + model_channels: int = 256 + conditioning_embedding_channels: int = None + rotary_interleaved: bool = True + layernorm_epsilon: float = 1e-06 + hidden_dropout: float = 0 + attention_dropout: float = 0 + add_qkv_bias: bool = True + use_cpu_initialization: bool = True + + load_from_flux_transformer: bool = True + guidance_scale: float = 3.5 + + data_step_fn: Callable = flux_controlnet_data_step + + +class FluxControlNet(VisionModule): + """ + A VisionModule-based neural network designed for Flux ControlNet tasks. + + + Args: + config (FluxControlNetConfig): + Configuration object containing model parameters such as input channels, hidden size, patch size, + and number of transformer layers. + """ + + def __init__(self, config: FluxControlNetConfig): + """ + Initializes the FluxControlNet model with embeddings, transformer layers, and optional conditioning blocks. + + Args: + config (FluxControlNetConfig): Configuration object with model parameters. + """ + super().__init__(config) + self.out_channels = config.in_channels + self.hidden_size = config.hidden_size + self.patch_size = config.patch_size + + self.pos_embed = EmbedND(dim=self.hidden_size, theta=10000, axes_dim=[16, 56, 56]) + self.img_embed = nn.Linear(config.in_channels, self.hidden_size) + self.txt_embed = nn.Linear(config.context_dim, self.hidden_size) + self.timestep_embedding = TimeStepEmbedder(config.model_channels, self.hidden_size) + self.vector_embedding = MLPEmbedder(in_dim=config.vec_in_dim, hidden_dim=self.hidden_size) + if config.guidance_embed: + self.guidance_embedding = ( + MLPEmbedder(in_dim=config.model_channels, hidden_dim=self.hidden_size) + if config.guidance_embed + else nn.Identity() + ) + self.double_blocks = nn.ModuleList( + [ + MMDiTLayer( + config=config, + submodules=get_flux_double_transformer_engine_spec().submodules, + layer_number=i, + context_pre_only=False, + ) + for i in range(config.num_joint_layers) + ] + ) + + self.single_blocks = nn.ModuleList( + [ + FluxSingleTransformerBlock( + config=config, + submodules=get_flux_single_transformer_engine_spec().submodules, + layer_number=i, + ) + for i in range(config.num_single_layers) + ] + ) + + # ContolNet Blocks + self.controlnet_double_blocks = nn.ModuleList() + for _ in range(config.num_joint_layers): + self.controlnet_double_blocks.append( + zero_module( + ColumnParallelLinear( + self.hidden_size, + self.hidden_size, + config=config, + init_method=nn.init.normal_, + gather_output=True, + ) + ) + ) + + self.controlnet_single_blocks = nn.ModuleList() + for _ in range(config.num_single_layers): + self.controlnet_single_blocks.append( + zero_module( + ColumnParallelLinear( + self.hidden_size, + self.hidden_size, + config=config, + init_method=nn.init.normal_, + gather_output=True, + ) + ) + ) + + if config.conditioning_embedding_channels is not None: + self.input_hint_block = ControlNetConditioningEmbedding( + conditioning_embedding_channels=config.conditioning_embedding_channels, + block_out_channels=(16, 16, 16, 16), + ) + self.controlnet_x_embedder = torch.nn.Linear(config.in_channels, self.hidden_size) + else: + self.input_hint_block = None + self.controlnet_x_embedder = zero_module(torch.nn.Linear(config.in_channels, self.hidden_size)) + + def load_from_flux_transformer(self, flux): + """ + Loads pre-trained weights from a Flux Transformer model into the FluxControlNet. + + Args: + flux (FluxTransformer): A pre-trained Flux Transformer model. + """ + logging.info("Loading ControlNet layer weights from Flux...") + self.pos_embed.load_state_dict(flux.pos_embed.state_dict()) + self.img_embed.load_state_dict(flux.img_embed.state_dict()) + self.txt_embed.load_state_dict(flux.txt_embed.state_dict()) + self.timestep_embedding.load_state_dict(flux.timestep_embedding.state_dict()) + self.vector_embedding.load_state_dict(flux.vector_embedding.state_dict()) + self.double_blocks.load_state_dict(flux.double_blocks.state_dict(), strict=False) + self.single_blocks.load_state_dict(flux.single_blocks.state_dict(), strict=False) + + def forward( + self, + img: torch.Tensor, + controlnet_cond: torch.Tensor, + txt: torch.Tensor = None, + y: torch.Tensor = None, + timesteps: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + conditioning_scale: float = 1.0, + ): + """ + Forward pass for the FluxControlNet model. + + Args: + img (torch.Tensor): Input image tensor. + controlnet_cond (torch.Tensor): Conditioning tensor for ControlNet. + txt (torch.Tensor, optional): Text embedding tensor. Default is None. + y (torch.Tensor, optional): Vector embedding tensor. Default is None. + timesteps (torch.LongTensor, optional): Time step tensor. Default is None. + img_ids (torch.Tensor, optional): Image IDs. Default is None. + txt_ids (torch.Tensor, optional): Text IDs. Default is None. + guidance (torch.Tensor, optional): Guidance tensor. Default is None. + conditioning_scale (float, optional): Scaling factor for conditioning. Default is 1.0. + + Returns: + torch.Tensor: The output of the forward pass. + """ + hidden_states = self.img_embed(img) + encoder_hidden_states = self.txt_embed(txt) + if self.input_hint_block is not None: + controlnet_cond = self.input_hint_block(controlnet_cond) + batch_size, channels, height_pw, width_pw = controlnet_cond.shape + height = height_pw // self.config.patch_size + width = width_pw // self.config.patch_size + controlnet_cond = controlnet_cond.reshape( + batch_size, channels, height, self.patch_size, width, self.patch_size + ) + controlnet_cond = controlnet_cond.permute(0, 2, 4, 1, 3, 5) + controlnet_cond = controlnet_cond.reshape(batch_size, height * width, -1) + + hidden_states = hidden_states + self.controlnet_x_embedder(controlnet_cond) + + timesteps = timesteps.to(img.dtype) * 1000 + vec_emb = self.timestep_embedding(timesteps) + if guidance is not None: + vec_emb = vec_emb + self.guidance_embedding(self.timestep_embedding.time_proj(guidance * 1000)) + vec_emb = vec_emb + self.vector_embedding(y) + + ids = torch.cat((txt_ids, img_ids), dim=1) + rotary_pos_emb = self.pos_embed(ids) + + double_block_samples = () + for id_block, block in enumerate(self.double_blocks): + hidden_states, encoder_hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + rotary_pos_emb=rotary_pos_emb, + emb=vec_emb, + ) + double_block_samples = double_block_samples + (hidden_states,) + + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=0) + + single_block_samples = () + for id_block, block in enumerate(self.single_blocks): + hidden_states = block( + hidden_states=hidden_states, + rotary_pos_emb=rotary_pos_emb, + emb=vec_emb, + ) + single_block_samples = single_block_samples + (hidden_states[encoder_hidden_states.shape[0] :, ...],) + + controlnet_double_block_samples = () + for double_block_sample, control_block in zip(double_block_samples, self.controlnet_double_blocks): + double_block_sample, bias = control_block(double_block_sample) + double_block_sample = double_block_sample + bias if bias else double_block_sample + controlnet_double_block_samples += (double_block_sample,) + + controlnet_single_block_samples = () + for single_block_sample, control_block in zip(single_block_samples, self.controlnet_single_blocks): + single_block_sample, bias = control_block(single_block_sample) + single_block_sample = single_block_sample + bias if bias else single_block_sample + controlnet_single_block_samples += (single_block_sample,) + + controlnet_double_block_samples = [sample * conditioning_scale for sample in controlnet_double_block_samples] + controlnet_single_block_samples = [sample * conditioning_scale for sample in controlnet_single_block_samples] + + controlnet_double_block_samples = ( + None if len(controlnet_double_block_samples) == 0 else controlnet_double_block_samples + ) + controlnet_single_block_samples = ( + None if len(controlnet_single_block_samples) == 0 else controlnet_single_block_samples + ) + + return controlnet_double_block_samples, controlnet_single_block_samples + + +class FluxControlnetForwardWrapper(VisionModule): + """ + A wrapper combines flux and flux controlnet forward pass for easier initialization. + """ + + def __init__(self, flux_config: FluxConfig, flux_controlnet_config: FluxControlNetConfig): + """ + Create flux and flux controlnet instances by their config. + """ + super().__init__(flux_config) + + self.flux = self.config.configure_model() + for param in self.flux.parameters(): + param.requires_grad = False + + self.flux_controlnet = FluxControlNet(flux_controlnet_config) + if flux_controlnet_config.load_from_flux_transformer: + self.flux_controlnet.load_from_flux_transformer(self.flux) + + +class MegatronFluxControlNetModel(MegatronFluxModel): + """ + Megatron wrapper for flux controlnet model. + + Args: + flux_params (FluxModelParams): Parameters to configure the Flux model. + flux_controlnet_config (FluxControlNetConfig): Configuration specific to the FluxControlNet. + + Methods: + configure_model: + Configures the model by wrapping the FluxControlNet with the appropriate layers and settings, + configuring the VAE, scheduler, and text encoders. + data_step: + A wrapper around the data-step function specific to FluxControlNet, controlling how data is processed. + forward: + Executes a forward pass through FluxControlNet. + training_step: + A wrapper step method that calls forward_step with a data batch from data loader. + forward_step: + Handles the forward pass specific to training, computing the model's output. + validation_step: + Calls inference pipeline with current model weights and save inference result together with the control + image. + """ + + def __init__(self, flux_params: FluxModelParams, flux_controlnet_config: FluxControlNetConfig): + super().__init__(flux_params) + self.flux_controlnet_config = flux_controlnet_config + self.optim.connect(self) + + def configure_model(self): + """ + Initialize flux and controlnet modules, vae, scheduler, and text encoders with given configs. + """ + if not hasattr(self, "module"): + self.module = FluxControlnetForwardWrapper(self.config, self.flux_controlnet_config) + + self.configure_vae(self.vae_config) + self.configure_scheduler() + self.configure_text_encoders(self.clip_params, self.t5_params) + # Have to disable requiring grads for those params not getting one, otherwise custom fsdp fails at assert + # when there is no single layer, encoder_hidden_states related params are not included in computation graph + for name, param in self.module.named_parameters(): + if self.flux_controlnet_config.num_single_layers == 0: + if "context" in name or "added" in name: + param.requires_grad = False + # When getting rid of concat, the projection bias in attention and mlp bias are identical + # So this bias is skipped and not included in the computation graph + if "single_blocks" in name and "self_attention.linear_proj.bias" in name: + param.requires_grad = False + + def data_step(self, dataloader_iter): + """ + Retrive data batch from dataloader iterator and do necessary processing before feeding into train steps. + """ + return self.flux_controlnet_config.data_step_fn(dataloader_iter) + + def forward(self, *args, **kwargs): + """ + Calling the controlnet forward pass. + """ + # FSDP module -> Bfloat16 module -> ForwardWrapper -> flux controlnet + return self.module.module.module.flux_controlnet(*args, **kwargs) + + def training_step(self, batch, batch_idx=None) -> torch.Tensor: + """ + A wrapper method takes data batch and returns the results of forward_step. + """ + # In mcore the loss-function is part of the forward-pass (when labels are provided) + return self.forward_step(batch) + + def forward_step(self, batch) -> torch.Tensor: + """ + The main forward step function. + """ + if self.optim.config.bf16: + self.autocast_dtype = torch.bfloat16 + elif self.optim.config.fp16: + self.autocast_dtype = torch.float + else: + self.autocast_dtype = torch.float32 + + if self.image_precached: + latents = batch["latents"].cuda(non_blocking=True) + control_latents = batch["control_latents"].cuda(non_blocking=True) + else: + img = batch["images"].cuda(non_blocking=True) + latents = self.vae.encode(img).to(dtype=self.autocast_dtype) + hint = batch["hint"].cuda(non_blocking=True) + control_latents = self.vae.encode(hint).to(dtype=self.autocast_dtype) + + latent_image_ids = self._prepare_latent_image_ids( + batch_size=latents.shape[0], + height=latents.shape[2], + width=latents.shape[3], + device=latents.device, + dtype=self.autocast_dtype, + ) + + latents = self._pack_latents( + latents, + batch_size=latents.shape[0], + num_channels_latents=latents.shape[1], + height=latents.shape[2], + width=latents.shape[3], + ) + + control_image = self._pack_latents( + control_latents, + batch_size=control_latents.shape[0], + num_channels_latents=control_latents.shape[1], + height=control_latents.shape[2], + width=control_latents.shape[3], + ).transpose(0, 1) + + batch_size = latents.shape[0] + noise = torch.randn_like(latents, device=latents.device, dtype=latents.dtype) + u = self.compute_density_for_timestep_sampling( + "logit_normal", + batch_size, + ) + indices = (u * self.scheduler.num_train_timesteps).long() + timesteps = self.scheduler.timesteps[indices].to(device=latents.device) + + sigmas = self.scheduler.sigmas.to(device=latents.device, dtype=latents.dtype) + schduler_timesteps = self.scheduler.timesteps.to(device=latents.device) + step_indices = [(schduler_timesteps == t).nonzero().item() for t in timesteps] + timesteps = timesteps.to(dtype=latents.dtype) + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < latents.ndim: + sigma = sigma.unsqueeze(-1) + packed_noisy_model_input = (1.0 - sigma) * latents + sigma * noise + packed_noisy_model_input = packed_noisy_model_input.transpose(0, 1) + + if self.config.guidance_embed: + guidance_vec = torch.full( + (packed_noisy_model_input.shape[1],), + self.config.guidance_scale, + device=latents.device, + dtype=latents.dtype, + ) + else: + guidance_vec = None + if self.text_precached: + prompt_embeds = batch["prompt_embeds"].cuda(non_blocking=True).transpose(0, 1) + pooled_prompt_embeds = batch["pooled_prompt_embeds"].cuda(non_blocking=True) + text_ids = batch["text_ids"].cuda(non_blocking=True) + else: + txt = batch["txt"] + prompt_embeds, pooled_prompt_embeds, text_ids = self.encode_prompt( + txt, device=latents.device, dtype=latents.dtype + ) + + with torch.cuda.amp.autocast( + self.autocast_dtype in (torch.half, torch.bfloat16), + dtype=self.autocast_dtype, + ): + controlnet_double_block_samples, controlnet_single_block_samples = self.forward( + img=packed_noisy_model_input, + controlnet_cond=control_image, + txt=prompt_embeds, + y=pooled_prompt_embeds, + timesteps=timesteps / 1000, + img_ids=latent_image_ids, + txt_ids=text_ids, + guidance=guidance_vec, + ) + noise_pred = self.module.module.module.flux( + img=packed_noisy_model_input, + txt=prompt_embeds, + y=pooled_prompt_embeds, + timesteps=timesteps / 1000, + img_ids=latent_image_ids, + txt_ids=text_ids, + guidance=guidance_vec, + controlnet_double_block_samples=controlnet_double_block_samples, + controlnet_single_block_samples=controlnet_single_block_samples, + ) + + target = (noise - latents).transpose(0, 1) + loss = F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + return loss + + def validation_step(self, batch, batch_idx=None): + """ + Initialize flux controlnet pipeline with current model components. + + Saves the inference results together with the hint image to log folder. + """ + logging.info("Start validation step") + from nemo.collections.diffusion.models.flux.pipeline import FluxControlNetInferencePipeline + + pipe = FluxControlNetInferencePipeline( + params=self.params, + contorlnet_config=self.flux_controlnet_config, + flux=self.module.module.module.flux, + vae=self.vae, + t5=self.t5, + clip=self.clip, + scheduler_steps=self.params.scheduler_steps, + flux_controlnet=self.module.module.module.flux_controlnet, + ) + + if self.image_precached and self.text_precached: + latents = batch["latents"].cuda(non_blocking=True) + control_latents = batch["control_latents"].cuda(non_blocking=True) + prompt_embeds = batch["prompt_embeds"].cuda(non_blocking=True).transpose(0, 1) + pooled_prompt_embeds = batch["pooled_prompt_embeds"].cuda(non_blocking=True) + log_images = pipe( + latents=latents, + control_image=control_latents, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + height=latents.shape[2] * self.vae_scale_factor, + width=latents.shape[3] * self.vae_scale_factor, + num_inference_steps=30, + num_images_per_prompt=1, + guidance_scale=7.0, + dtype=self.autocast_dtype, + save_to_disk=False, + ) + log_images[0].save(f"{self.logger.log_dir}/step={self.global_step}_rank{self.local_rank}.png") + else: + img = batch["images"].cuda(non_blocking=True) + hint = batch["hint"].cuda(non_blocking=True) + text = batch["txt"] + log_images = pipe( + text, + control_image=hint, + num_inference_steps=30, + num_images_per_prompt=1, + height=img.shape[2], + width=img.shape[3], + guidance_scale=7.0, + dtype=self.autocast_dtype, + save_to_disk=False, + ) + log_images[0].save(f"{self.logger.log_dir}/step={self.global_step}_rank{self.local_rank}_{text}.png") + hint = pipe.torch_to_numpy(hint) + hint = pipe.numpy_to_pil(hint) + hint[0].save(f"{self.logger.log_dir}/step={self.global_step}_rank{self.local_rank}_control.png") + return torch.tensor([0.0], device=torch.cuda.current_device()) diff --git a/nemo_vfm/diffusion/models/model.py b/nemo_vfm/diffusion/models/model.py new file mode 100755 index 00000000..bb19d08e --- /dev/null +++ b/nemo_vfm/diffusion/models/model.py @@ -0,0 +1,855 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import importlib +import os +import warnings +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +import wandb +from einops import rearrange +from megatron.core import parallel_state +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.transformer_config import TransformerConfig +from nemo.collections.diffusion.models.dit_llama.dit_llama_model import DiTLlamaModel +from nemo.collections.diffusion.sampler.conditioner import ( + VideoConditioner, + VideoExtendConditioner, + VideoExtendConditionerControl, +) +from nemo.collections.diffusion.sampler.conditioner_configs import ( + ActionControlConfig, + FPSConfig, + ImageSizeConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, + VideoCondBoolConfig, +) +from nemo.collections.diffusion.sampler.cosmos.cosmos_control_diffusion_pipeline import CosmosControlDiffusionPipeline +from nemo.collections.diffusion.sampler.cosmos.cosmos_diffusion_pipeline import CosmosDiffusionPipeline +from nemo.collections.diffusion.sampler.cosmos.cosmos_extended_diffusion_pipeline import ExtendedDiffusionPipeline +from nemo.collections.diffusion.sampler.edm.edm_pipeline import EDMPipeline +from nemo.collections.llm.gpt.model.base import GPTModel +from nemo.lightning import io +from nemo.lightning.megatron_parallel import MaskedTokenLossReduction, MegatronLossReduction +from nemo.lightning.pytorch.optim import OptimizerModule +from torch import nn +from typing_extensions import override + +from .dit.dit_model import DiTCrossAttentionModel +from .dit.dit_model_7b import ( + DiTControl7B, + DiTCrossAttentionActionExtendModel7B, + DiTCrossAttentionExtendModel7B, + DiTCrossAttentionModel7B, + PatchEmbed, +) +from .dit.dit_model_14b import ( + DiTCrossAttentionActionExtendModel14B, + DiTCrossAttentionExtendModel14B, + DiTCrossAttentionModel14B, +) + + +def dit_forward_step(model, batch) -> torch.Tensor: + return model(**batch) + + +def dit_data_step(module, dataloader_iter): + batch = next(dataloader_iter)[0] + batch = get_batch_on_this_cp_rank(batch) + batch = {k: v.to(device="cuda", non_blocking=True) if torch.is_tensor(v) else v for k, v in batch.items()} + batch["is_preprocessed"] = True # assume data is preprocessed + + if ("seq_len_q" in batch) and ("seq_len_kv" in batch): + cu_seqlens = batch["seq_len_q"].cumsum(dim=0).to(torch.int32) + zero = torch.zeros(1, dtype=torch.int32, device="cuda") + cu_seqlens = torch.cat((zero, cu_seqlens)) + + cu_seqlens_kv = batch["seq_len_kv"].cumsum(dim=0).to(torch.int32) + cu_seqlens_kv = torch.cat((zero, cu_seqlens_kv)) + + batch["packed_seq_params"] = { + "self_attention": PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + qkv_format=module.qkv_format, + ), + "cross_attention": PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens_kv, + qkv_format=module.qkv_format, + ), + } + + return batch + + +def get_batch_on_this_cp_rank(data: Dict): + """Split the data for context parallelism.""" + from megatron.core import mpu + + cp_size = mpu.get_context_parallel_world_size() + cp_rank = mpu.get_context_parallel_rank() + + t = 16 + if cp_size > 1: + # cp split on seq_length, for video_latent, noise_latent and pos_ids + assert t % cp_size == 0, "t must divisibly by cp_size" + num_valid_tokens_in_ub = None + if "loss_mask" in data and data["loss_mask"] is not None: + num_valid_tokens_in_ub = data["loss_mask"].sum() + + for key, value in data.items(): + if (value is not None) and (key in ["video", "video_latent", "noise_latent", "pos_ids"]): + if len(value.shape) > 5: + value = value.squeeze(0) + B, C, T, H, W = value.shape + if T % cp_size == 0: + # FIXME packed sequencing + data[key] = value.view(B, C, cp_size, T // cp_size, H, W)[:, :, cp_rank, ...].contiguous() + else: + # FIXME packed sequencing + data[key] = value.view(B, C, T, cp_size, H // cp_size, W)[:, :, :, cp_rank, ...].contiguous() + loss_mask = data["loss_mask"] + data["loss_mask"] = loss_mask.view(loss_mask.shape[0], cp_size, loss_mask.shape[1] // cp_size)[ + :, cp_rank, ... + ].contiguous() + data["num_valid_tokens_in_ub"] = num_valid_tokens_in_ub + + return data + + +@dataclass +class DiTConfig(TransformerConfig, io.IOMixin): + """ + Config for DiT-S model + """ + + crossattn_emb_size: int = 1024 + add_bias_linear: bool = False + gated_linear_unit: bool = False + + num_layers: int = 12 + hidden_size: int = 384 + max_img_h: int = 80 + max_img_w: int = 80 + max_frames: int = 34 + patch_spatial: int = 2 + num_attention_heads: int = 6 + layernorm_epsilon = 1e-6 + normalization = "RMSNorm" + add_bias_linear = False + qk_layernorm_per_head = True + layernorm_zero_centered_gamma = False + + fp16_lm_cross_entropy: bool = False + parallel_output: bool = True + share_embeddings_and_output_weights: bool = True + + # max_position_embeddings: int = 5400 + hidden_dropout: float = 0 + attention_dropout: float = 0 + + bf16: bool = True + params_dtype: torch.dtype = torch.bfloat16 + + vae_module: str = "nemo.collections.diffusion.vae.diffusers_vae.AutoencoderKLVAE" + vae_path: str = None + sigma_data: float = 0.5 + + in_channels: int = 16 + + data_step_fn = dit_data_step + forward_step_fn = dit_forward_step + + replicated_t_embedder = True + + @override + def configure_model(self, tokenizer=None) -> DiTCrossAttentionModel: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + if isinstance(self, DiTLlama30BConfig): + model = DiTLlamaModel + elif isinstance(self, DiT14BConfig): + model = DiTCrossAttentionModel14B + return model( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + elif isinstance(self, DiT7BConfig): + model = DiTCrossAttentionModel7B + return model( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + else: + model = DiTCrossAttentionModel + + return model( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + max_img_h=self.max_img_h, + max_img_w=self.max_img_w, + max_frames=self.max_frames, + patch_spatial=self.patch_spatial, + ) + + def configure_vae(self): + return dynamic_import(self.vae_module)(self.vae_path) + + +@dataclass +class DiTBConfig(DiTConfig): + num_layers: int = 12 + hidden_size: int = 768 + num_attention_heads: int = 12 + + +@dataclass +class DiTLConfig(DiTConfig): + num_layers: int = 24 + hidden_size: int = 1024 + num_attention_heads: int = 16 + + +@dataclass +class DiTXLConfig(DiTConfig): + num_layers: int = 28 + hidden_size: int = 1152 + num_attention_heads: int = 16 + + +@dataclass +class DiT7BConfig(DiTConfig): + num_layers: int = 28 + hidden_size: int = 4096 + crossattn_emb_size: int = 1024 + max_img_h: int = 240 + max_img_w: int = 240 + max_frames: int = 128 + patch_spatial: int = 2 + num_attention_heads: int = 32 + layernorm_epsilon = 1e-6 + normalization = "RMSNorm" + gated_linear_unit: bool = False + add_bias_linear: bool = False + qk_layernorm_per_head: bool = True + apply_rope_fusion: bool = True + layernorm_zero_centered_gamma: bool = False + additional_timestamp_channels = None + fp16_lm_cross_entropy: bool = False + parallel_output: bool = True + share_embeddings_and_output_weights: bool = True + hidden_dropout: float = 0 + attention_dropout: float = 0 + bf16: bool = True + params_dtype: torch.dtype = torch.bfloat16 + vae_module: str = "nemo.collections.diffusion.vae.video_vae.video_vae3_512" + vae_path: str = None + sigma_data: float = 0.5 + loss_add_logvar: bool = True + data_step_fn = dit_data_step + forward_step_fn = dit_forward_step + model_name: str = "cosmos_7b_text2world" + + +@dataclass +class DiT7BVideo2WorldConfig(DiT7BConfig): + model_name: str = "cosmos_7b_video2world" + + @override + def configure_model(self, tokenizer=None) -> DiTCrossAttentionExtendModel7B: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + model = DiTCrossAttentionExtendModel7B + return model( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + + +@dataclass +class DiT7BControl2WorldConfig(DiT7BConfig): + model_name: str = "cosmos_7b_control2world" + + # num_layers: int = + @override + def configure_model(self, tokenizer=None) -> DiTControl7B: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + model = DiTControl7B + return model( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + + +@dataclass +class DiT7BCameraCtrlConfig(DiT7BConfig): + model_name: str = "cosmos_7b_video2world_camera_ctrl" + + @override + def configure_vae(self): + return dynamic_import(self.vae_module)(self.vae_path, pixel_chunk_duration=57) + + @override + def configure_model(self, tokenizer=None) -> DiTCrossAttentionExtendModel7B: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + dit_model = DiTCrossAttentionExtendModel7B( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + dit_model.x_embedder = ( + PatchEmbed( + spatial_patch_size=dit_model.patch_spatial, + temporal_patch_size=dit_model.patch_temporal, + in_channels=24, + out_channels=dit_model.config.hidden_size, + bias=False, + keep_spatio=True, + legacy_patch_emb=dit_model.legacy_patch_emb, + ) + .cuda() + .to(dtype=torch.bfloat16) + ) + dit_model.extra_per_block_abs_pos_emb = True + return dit_model + + +@dataclass +class DiT7BActionConfig(DiTConfig): + num_layers: int = int(os.getenv("NUM_LAYERS", 28)) + hidden_size: int = 4096 + crossattn_emb_size: int = 1024 + max_img_h: int = 240 + max_img_w: int = 240 + max_frames: int = 128 + patch_spatial: int = 2 + num_attention_heads: int = 32 + layernorm_epsilon = 1e-6 + normalization = "RMSNorm" + gated_linear_unit: bool = False + add_bias_linear: bool = False + qk_layernorm_per_head: bool = True + apply_rope_fusion: bool = True + layernorm_zero_centered_gamma: bool = False + additional_timestamp_channels = None + fp16_lm_cross_entropy: bool = False + parallel_output: bool = True + share_embeddings_and_output_weights: bool = True + hidden_dropout: float = 0 + attention_dropout: float = 0 + bf16: bool = True + params_dtype: torch.dtype = torch.bfloat16 + vae_module: str = "nemo.collections.diffusion.vae.video_vae.video_vae3_512" + vae_path: str = None + sigma_data: float = 0.5 + loss_add_logvar: bool = True + data_step_fn = dit_data_step + forward_step_fn = dit_forward_step + model_name: str = "cosmos_7b_video2world_action_control" + recompute_granularity: str = "full" + recompute_method: str = "uniform" + recompute_num_layers: int = 1 + action_emb_dim: int = 7 + + +@dataclass +class DiT7BVideo2WorldActionConfig(DiT7BActionConfig): + @override + def configure_model(self, tokenizer=None) -> DiTCrossAttentionActionExtendModel7B: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + return DiTCrossAttentionActionExtendModel7B( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + + +@dataclass +class DiT14BConfig(DiTConfig): + num_layers: int = 36 + hidden_size: int = 5120 + crossattn_emb_size: int = 1024 + max_img_h: int = 240 + max_img_w: int = 240 + max_frames: int = 128 + patch_spatial: int = 2 + num_attention_heads: int = 40 + layernorm_epsilon = 1e-6 + normalization = "RMSNorm" + gated_linear_unit: bool = False + add_bias_linear: bool = False + qk_layernorm_per_head: bool = True + apply_rope_fusion: bool = True + layernorm_zero_centered_gamma: bool = False + additional_timestamp_channels = None + fp16_lm_cross_entropy: bool = False + parallel_output: bool = True + share_embeddings_and_output_weights: bool = True + hidden_dropout: float = 0 + attention_dropout: float = 0 + bf16: bool = True + params_dtype: torch.dtype = torch.bfloat16 + vae_module: str = "nemo.collections.diffusion.vae.video_vae.video_vae3_512" + vae_path: str = None + sigma_data: float = 0.5 + loss_add_logvar: bool = True + data_step_fn = dit_data_step + forward_step_fn = dit_forward_step + model_name = "cosmos_14b_text2world" + + +@dataclass +class DiT14BVideo2WorldConfig(DiT14BConfig): + model_name = "cosmos_14b_video2world" + + @override + def configure_model(self, tokenizer=None) -> DiTCrossAttentionExtendModel14B: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + model = DiTCrossAttentionExtendModel14B + return model( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + + +@dataclass +class DiT14BActionConfig(DiTConfig): + num_layers: int = int(os.getenv("NUM_LAYERS", 36)) + hidden_size: int = 5120 + crossattn_emb_size: int = 1024 + max_img_h: int = 240 + max_img_w: int = 240 + max_frames: int = 128 + patch_spatial: int = 2 + num_attention_heads: int = 40 + layernorm_epsilon = 1e-6 + normalization = "RMSNorm" + gated_linear_unit: bool = False + add_bias_linear: bool = False + qk_layernorm_per_head: bool = True + apply_rope_fusion: bool = True + layernorm_zero_centered_gamma: bool = False + additional_timestamp_channels = None + fp16_lm_cross_entropy: bool = False + parallel_output: bool = True + share_embeddings_and_output_weights: bool = True + hidden_dropout: float = 0 + attention_dropout: float = 0 + bf16: bool = True + params_dtype: torch.dtype = torch.bfloat16 + vae_module: str = "nemo.collections.diffusion.vae.video_vae.video_vae3_512" + vae_path: str = None + sigma_data: float = 0.5 + loss_add_logvar: bool = True + data_step_fn = dit_data_step + forward_step_fn = dit_forward_step + model_name: str = "cosmos_14b_video2world_action_control" + recompute_granularity: str = "full" + recompute_method: str = "uniform" + recompute_num_layers: int = 1 + action_emb_dim: int = 7 + + +@dataclass +class DiT14BVideo2WorldActionConfig(DiT14BActionConfig): + @override + def configure_model(self, tokenizer=None) -> DiTCrossAttentionActionExtendModel14B: + vp_size = self.virtual_pipeline_model_parallel_size + if vp_size: + p_size = self.pipeline_model_parallel_size + assert (self.num_layers // p_size) % vp_size == 0, ( + "Make sure the number of model chunks is the same across all pipeline stages." + ) + + return DiTCrossAttentionActionExtendModel14B( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + ) + + +@dataclass +class DiTLlama30BConfig(DiTConfig): + num_layers: int = 48 + hidden_size: int = 6144 + ffn_hidden_size: int = 16384 + num_attention_heads: int = 48 + num_query_groups: int = 8 + gated_linear_unit: int = True + bias_activation_fusion: int = True + activation_func: Callable = F.silu + normalization: str = "RMSNorm" + layernorm_epsilon: float = 1e-5 + max_frames: int = 128 + max_img_h: int = 240 + max_img_w: int = 240 + patch_spatial: int = 2 + + init_method_std: float = 0.01 + add_bias_linear: bool = False + seq_length: int = 256 + + bias_activation_fusion: bool = True + masked_softmax_fusion: bool = True + persist_layer_norm: bool = True + bias_dropout_fusion: bool = True + + +@dataclass +class DiTLlama5BConfig(DiTLlama30BConfig): + num_layers: int = 32 + hidden_size: int = 3072 + ffn_hidden_size: int = 8192 + num_attention_heads: int = 24 + + +class DiTModel(GPTModel): + def __init__( + self, + config: Optional[DiTConfig] = None, + optim: Optional[OptimizerModule] = None, + model_transform: Optional[Callable[[nn.Module], nn.Module]] = None, + tokenizer: Optional[Any] = None, + ): + super().__init__(config or DiTConfig(), optim=optim, model_transform=model_transform) + + self.vae = None + self._training_loss_reduction = None + self._validation_loss_reduction = None + + if hasattr(config, "model_name"): + model_name = getattr(config, "model_name") + print(f"Initializing the DiT model with configuration model name: [{model_name}]") + if "cosmos" in model_name and "text2world" in model_name: + self.conditioner = VideoConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + ) + self.diffusion_pipeline = CosmosDiffusionPipeline( + net=self, + conditioner=self.conditioner, + loss_add_logvar=self.config.loss_add_logvar, + ) + elif "cosmos" in model_name and "video2world" in model_name: + # Parse conditioner configs. + conditioner_configs = { + "text": TextConfig(), + "fps": FPSConfig(), + "num_frames": NumFramesConfig(), + "image_size": ImageSizeConfig(), + "padding_mask": PaddingMaskConfig(), + "video_cond_bool": VideoCondBoolConfig( + condition_location="first_random_n", + apply_corruption_to_condition_region="noise_with_sigma", + ), + } + if "action_control" in model_name: + # Add an action control tensor input. + conditioner_configs["action_ctrl"] = ActionControlConfig() + elif "camera_ctrl" in model_name: + # Overwrite the default 'video_cond_bool' condition + conditioner_configs["video_cond_bool"] = VideoCondBoolConfig( + condition_location="first_random_n", + apply_corruption_to_condition_region="noise_with_sigma", + add_pose_condition=True, + first_random_n_num_condition_t_max=1, + ) + # Instantiate VideoExtendConditioner and ExtendedDiffusionPipeline. + self.conditioner = VideoExtendConditioner(**conditioner_configs) + self.diffusion_pipeline = ExtendedDiffusionPipeline( + net=self, + conditioner=self.conditioner, + loss_add_logvar=self.config.loss_add_logvar, + ) + elif "cosmos" in model_name and "control2world" in model_name: + conditioner_configs = { + "text": TextConfig(), + "fps": FPSConfig(), + "num_frames": NumFramesConfig(), + "image_size": ImageSizeConfig(), + "padding_mask": PaddingMaskConfig(), + "video_cond_bool": VideoCondBoolConfig( + condition_location="first_random_n", + apply_corruption_to_condition_region="noise_with_sigma", + ), + } + self.conditioner = VideoExtendConditionerControl(**conditioner_configs) + self.diffusion_pipeline = CosmosControlDiffusionPipeline( + net=self, + base_model=None, # initialize as empty for now + conditioner=self.conditioner, + loss_add_logvar=self.config.loss_add_logvar, + ) + else: + self.diffusion_pipeline = EDMPipeline(net=self, sigma_data=self.config.sigma_data) + + def compute_logvar(self, c_noise): + logvar = self.module.module.module.logvar + return logvar(c_noise) + + def data_step(self, dataloader_iter) -> Dict[str, Any]: + return self.config.data_step_fn(dataloader_iter) + + def forward(self, *args, **kwargs): + return self.module.forward(*args, **kwargs) + + def forward_step(self, batch) -> torch.Tensor: + if hasattr(self.config, "model_name"): + if "cosmos" in getattr(self.config, "model_name"): + data_batch = { + k: v.to(device="cuda", non_blocking=True) if torch.is_tensor(v) else v for k, v in batch.items() + } + data_batch["is_preprocessed"] = True # assume data is preprocessed + if parallel_state.is_pipeline_last_stage(): + output_batch, kendall_loss = self.diffusion_pipeline.training_step(batch, 0) + if torch.distributed.get_rank() == 0 and wandb.run: + wandb.log( + {k: output_batch[k] for k in ["edm_loss"] if k in output_batch}, step=self.global_step + ) + kendall_loss = torch.mean(kendall_loss, dim=1) + return kendall_loss + else: + output_tensor = self.diffusion_pipeline.training_step(batch, 0) + return output_tensor + + else: + if parallel_state.is_pipeline_last_stage(): + output_batch, loss = self.diffusion_pipeline.training_step(batch, 0) + loss = torch.mean(loss, dim=1) + return loss + else: + output_tensor = self.diffusion_pipeline.training_step(batch, 0) + return output_tensor + + def training_step(self, batch, batch_idx=None) -> torch.Tensor: + # In mcore the loss-function is part of the forward-pass (when labels are provided) + return self.forward_step(batch) + + def on_validation_start(self): + if self.vae is None: + if self.config.vae_path is None: + warnings.warn("vae_path not specified skipping validation") + return None + self.vae = self.config.configure_vae() + self.vae.to("cuda") + + def on_validation_end(self): + if self.vae is not None: + self.vae.to("cpu") + + def validation_step(self, batch, batch_idx=None) -> torch.Tensor: + # In mcore the loss-function is part of the forward-pass (when labels are provided) + state_shape = batch["video"].shape + sample = self.diffusion_pipeline.generate_samples_from_batch( + batch, + guidance=7, + state_shape=state_shape, + num_steps=35, + is_negative_prompt=True if "neg_t5_text_embeddings" in batch else False, + ) + + # TODO visualize more than 1 sample + sample = sample[0, None] + C, T, H, W = batch["latent_shape"][0] + seq_len_q = batch["seq_len_q"][0] + + sample = rearrange( + sample[:, :seq_len_q], + "B (T H W) (ph pw pt C) -> B C (T pt) (H ph) (W pw)", + ph=self.config.patch_spatial, + pw=self.config.patch_spatial, + C=C, + T=T, + H=H // self.config.patch_spatial, + W=W // self.config.patch_spatial, + ) + + video = (1.0 + self.vae.decode(sample / self.config.sigma_data)).clamp(0, 2) / 2 # [B, 3, T, H, W] + + video = (video * 255).to(torch.uint8).cpu().numpy().astype(np.uint8) + + T = video.shape[2] + if T == 1: + image = rearrange(video, "b c t h w -> (b t h) w c") + result = image + else: + # result = wandb.Video(video, fps=float(batch['fps'])) # (batch, time, channel, height width) + result = video + + # wandb is on the last rank for megatron, first rank for nemo + wandb_rank = 0 + + if parallel_state.get_data_parallel_src_rank() == wandb_rank: + if torch.distributed.get_rank() == wandb_rank: + gather_list = [None for _ in range(parallel_state.get_data_parallel_world_size())] + else: + gather_list = None + torch.distributed.gather_object( + result, gather_list, wandb_rank, group=parallel_state.get_data_parallel_group() + ) + if gather_list is not None: + videos = [] + for video in gather_list: + if len(video.shape) == 3: + videos.append(wandb.Image(video)) + else: + videos.append(wandb.Video(video, fps=30)) + wandb.log({"prediction": videos}, step=self.global_step) + + return None + + @property + def training_loss_reduction(self) -> MaskedTokenLossReduction: + if not self._training_loss_reduction: + self._training_loss_reduction = MaskedTokenLossReduction() + + return self._training_loss_reduction + + @property + def validation_loss_reduction(self) -> MaskedTokenLossReduction: + if not self._validation_loss_reduction: + self._validation_loss_reduction = DummyLossReduction() + + return self._validation_loss_reduction + + def on_validation_model_zero_grad(self) -> None: + """ + Small hack to avoid first validation on resume. + This will NOT work if the gradient accumulation step should be performed at this point. + https://github.com/Lightning-AI/pytorch-lightning/discussions/18110 + """ + super().on_validation_model_zero_grad() + if self.trainer.ckpt_path is not None and getattr(self, "_restarting_skip_val_flag", True): + self.trainer.sanity_checking = True + self._restarting_skip_val_flag = False + + +class DummyLossReduction(MegatronLossReduction): + def __init__(self, validation_step: bool = False, val_drop_last: bool = True) -> None: + super().__init__() + self.validation_step = validation_step + self.val_drop_last = val_drop_last + + def forward( + self, batch: Dict[str, torch.Tensor], forward_out: torch.Tensor + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + return torch.tensor(0.0, device=torch.cuda.current_device()), { + "avg": torch.tensor(0.0, device=torch.cuda.current_device()) + } + + def reduce(self, losses_reduced_per_micro_batch) -> torch.Tensor: + return torch.tensor(0.0, device=torch.cuda.current_device()) + + +def dynamic_import(full_path): + """ + Dynamically import a class or function from a given full path. + + :param full_path: The full path to the class or function (e.g., "package.module.ClassName") + :return: The imported class or function + :raises ImportError: If the module or attribute cannot be imported + :raises AttributeError: If the attribute does not exist in the module + """ + try: + # Split the full path into module path and attribute name + module_path, attribute_name = full_path.rsplit(".", 1) + except ValueError as e: + raise ImportError( + f"Invalid full path '{full_path}'. It should contain both module and attribute names." + ) from e + + # Import the module + try: + module = importlib.import_module(module_path) + except ImportError as e: + raise ImportError(f"Cannot import module '{module_path}'.") from e + + # Retrieve the attribute from the module + try: + attribute = getattr(module, attribute_name) + except AttributeError as e: + raise AttributeError(f"Module '{module_path}' does not have an attribute '{attribute_name}'.") from e + + return attribute diff --git a/nemo_vfm/diffusion/models/moviegen/moviegen_model.py b/nemo_vfm/diffusion/models/moviegen/moviegen_model.py new file mode 100644 index 00000000..e9a3db00 --- /dev/null +++ b/nemo_vfm/diffusion/models/moviegen/moviegen_model.py @@ -0,0 +1,172 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116 + +from typing import Literal + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from megatron.core import tensor_parallel +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.transformer_config import TransformerConfig +from nemo.collections.diffusion.models.dit.dit_model import DiTCrossAttentionModel +from nemo.collections.diffusion.models.dit_llama.dit_llama_layer_spec import get_dit_llama_spec +from torch import Tensor + + +class TransposedLinear(nn.Module): + def __init__(self, in_features, out_features): + super(TransposedLinear, self).__init__() + # Note the swapped dimensions + self.weight = nn.Parameter(torch.Tensor(in_features, out_features)) + + def forward(self, input): + # Transpose the weight back in the forward pass + return F.linear(input, self.weight.t()) + + +class DiTLlamaModel(DiTCrossAttentionModel): + def __init__( + self, + config: TransformerConfig, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + position_embedding_type: Literal["learned_absolute", "rope"] = "rope", + max_img_h: int = 80, + max_img_w: int = 80, + max_frames: int = 34, + patch_spatial: int = 1, + patch_temporal: int = 1, + in_channels: int = 16, + out_channels: int = 16, + **kwargs, + ): + super().__init__( + config=config, + pre_process=pre_process, + post_process=post_process, + fp16_lm_cross_entropy=fp16_lm_cross_entropy, + parallel_output=parallel_output, + position_embedding_type=position_embedding_type, + max_img_h=max_img_h, + max_img_w=max_img_w, + max_frames=max_frames, + patch_spatial=patch_spatial, + patch_temporal=patch_temporal, + in_channels=in_channels, + out_channels=out_channels, + transformer_decoder_layer_spec=get_dit_llama_spec, + **kwargs, + ) + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + B = x.shape[0] + fps = kwargs.get( + "fps", + torch.tensor( + [ + 30, + ] + * B, + dtype=torch.bfloat16, + ), + ).view(-1) + pos_emb = None + if self.pre_process: + # transpose to match + x_B_S_D = self.x_embedder(x) + pos_emb = self.pos_embedder(pos_ids) + pos_emb = rearrange(pos_emb, "B S D -> S B D") + x_S_B_D = rearrange(x_B_S_D, "B S D -> S B D") + else: + # intermediate stage of pipeline + x_S_B_D = None ### should it take encoder_hidden_states + + timesteps_B_D = self.t_embedder(timesteps.flatten()).to(torch.bfloat16) # (b d_text_embedding) + + affline_emb_B_D = timesteps_B_D + fps_B_D = self.fps_embedder(fps) + fps_B_D = nn.functional.pad(fps_B_D, (0, self.config.hidden_size - fps_B_D.shape[1])) + affline_emb_B_D += fps_B_D + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + crossattn_emb = crossattn_emb.clone() + + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + rotary_pos_emb=pos_emb, + packed_seq_params=packed_seq_params, + ) + + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_S_B_D = self.final_layer_linear(x_S_B_D) + return rearrange(x_S_B_D, "S B D -> B S D") + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Sets input tensor to the model. + + See megatron.model.transformer.set_input_tensor() + + Args: + input_tensor (Tensor): Sets the input tensor for the model. + """ + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + assert len(input_tensor) == 1, "input_tensor should only be length 1 for gpt/bert" + self.decoder.set_input_tensor(input_tensor[0]) diff --git a/nemo_vfm/diffusion/readme.rst b/nemo_vfm/diffusion/readme.rst new file mode 100644 index 00000000..87152794 --- /dev/null +++ b/nemo_vfm/diffusion/readme.rst @@ -0,0 +1,190 @@ +Diffusion Training Framework +============= + +Overview +-------- + +The NeMo Diffusion Training Framework provides a scalable training platform for diffusion models with transformer backbones. Our new features streamline the training process, allowing developers to efficiently train state-of-the-art models with ease. + + +Some of the features we currently support include: + +- Energon Dataloader for Webscale Dataloading +- Model and Data Parallelism +- Model Architectures: DiT 30B parameters or even more + + +Features Status +--------------- + +We support image diffusion training. Video training incoming. + + ++---------------------------+------------------+ +| Parallelism | Status | ++===========================+==================+ +| FSDP | ✅ Supported | ++---------------------------+------------------+ +| CP+TP+SP+distopt | ✅ Supported | ++---------------------------+------------------+ +| CP+TP+SP+PP+distopt | ✅ Supported | ++---------------------------+------------------+ +| CP+TP+SP+FSDP | 🕒 Coming Soon | ++---------------------------+------------------+ + + +**Legend:** +- **FSDP**: Fully Sharded Data Parallelism +- **CP**: Context Parallelism +- **TP**: Tensor Parallelism +- **SP**: Sequence Parallelism +- **PP**: Pipeline Parallelism +- **distop**: mcore distributed optmizer + ++--------------+-------------------+-----------------+ +| Model Size | Modality | Status | ++==============+===================+=================+ +| DiT 30B+ | 256px image | ✅ Supported | ++--------------+-------------------+-----------------+ +| DiT 30B+ | 256px image+video | 🕒 Coming Soon | ++--------------+-------------------+-----------------+ +| DiT 30B+ | 768px image+video | 🕒 Coming Soon | ++--------------+-------------------+-----------------+ + + +Energon Dataloader for Webscale Dataloading +------------------------------------------- + +Webscale Dataloading +^^^^^^^^^^^^^^^^^^^^ + +Megatron-Energon is an optimized multi-modal dataloader for large-scale deep learning with Megatron. Energon allows for distributed loading of large training training data for multi-modal model training. Energon allows for blending many datasets together and distributing the dataloading workflow across multiple cluster nodes/processes while ensuring reproducibility and resumability. + +Dataloader Checkpointing +^^^^^^^^^^^^^^^^^^^^^^^^ + +One of Energon's key features is its ability to save and restore its state. This functionality is crucial for long-running training processes, making the dataloader robust and recoverable after interruptions. By allowing checkpointing of the dataloader status, Energon ensures that training can be resumed from where it left off, saving time and computational resources in case of unexpected shutdowns or planned pauses in the training process. This makes it especially useful for large scale training as it requires several training jobs for end-to-end training. + +Parallel Configuration +^^^^^^^^^^^^^^^^^^^^^^ + +Energon's architecture allows it to efficiently distribute data across multiple processing units, ensuring that each GPU or node receives a balanced workload. This parallelization not only increases the overall throughput of data processing but also helps in maintaining high utilization of available computational resources. + + +Mixed Image-Video Training (comming soon) +------------------------------ + +Our dataloader provides support for mixed image-video training by using the NeMo packed sequence feature to pack together images and videos of varying length into the same microbatch. The sequence packing mechanism uses the THD attention kernel, which allows us to increase the model FLOPs utilization (MFU) and efficiently process data with varying length. + + +.. image:: assets/mixed_training.png + :alt: Mixed image-video dataloading strategy + :width: 300px + :align: center + +Model and Data Parallelism +-------------------------- +NeMo provides support for training models using tensor parallelism, sequence parallelism, pipeline parallelism, and context parallelism. To support pipeline parallelism with conditional diffusion training, we duplicate the conditional embeddings across the pipeline stages, and perform an all-reduce during the backward pass. This approach uses more compute, but it has a lower communication cost than sending the conditional embeddings through different pipeline stages. + +.. image:: assets/pipeline_conditioning.png + :alt: Conditioning mechanism for pipeline parallelism + :width: 300px + :align: center + +Model Architectures +------------------- + +DiT +^^^ +We implement an efficient version of the diffusion transformer (DiT) [1]_. Our DiT is slightly modified from the original paper as we use cross attention and adaptive layernorm together in the same architecture. We also use a QK-layernorm for training stability. Our framework allows for customizing the DiT architecture while maintaining its scalability, enabling training large DiT models on long sequence lengths. + + + +Data preparation +-------------------------- + +We expect data to be in this webdataset format. For more information about webdataset and energon dataset, please refer to https://github.com/NVIDIA/Megatron-Energon + +Here we demonstrate a step by step example of how to prepare a dummy image dataset. + +.. code-block:: bash + + torchrun --nproc-per-node 2 nemo/collections/diffusion/data/prepare_energon_dataset.py --factory prepare_dummy_image_dataset + +this will generate a folder a tar files. .pth contains image/video latent representations encode by image/video tokenizer, .json contains metadata including text caption, resolution, aspection ratio, and .pickle contains text embeddings encoded by language model like T5. + +.. code-block:: bash + + shard_000.tar + ├── samples/sample_0000.pth + ├── samples/sample_0000.pickle + ├── samples/sample_0000.json + ├── samples/sample_0001.pth + ├── samples/sample_0001.pickle + ├── samples/sample_0001.json + └── ... + shard_001.tar + +The following is a sample command to prepare prepare webdataset into energon dataset: + +.. code-block:: bash + + # energon prepare . --num-workers 192 + Found 369057 tar files in total. The first and last ones are: + - 0.tar + - 99999.tar + If you want to exclude some of them, cancel with ctrl+c and specify an exclude filter in the command line. + Please enter a desired train/val/test split like "0.5, 0.2, 0.3" or "8,1,1": 1,0,0 + Indexing shards [####################################] 369057/369057 + Sample 0, keys: + - .json + - .pickle + - .pth + Sample 1, keys: + - .json + - .pickle + - .pth + Found the following part types in the dataset: .json, .pth, .pickle + Do you want to create a dataset.yaml interactively? [Y/n]: Y + The following dataset classes are available: + 0. CaptioningWebdataset + 1. CrudeWebdataset + 2. ImageClassificationWebdataset + 3. ImageWebdataset + 4. InterleavedWebdataset + 5. MultiChoiceVQAWebdataset + 6. OCRWebdataset + 7. SimilarityInterleavedWebdataset + 8. TextWebdataset + 9. VQAOCRWebdataset + 10. VQAWebdataset + 11. VidQAWebdataset + Please enter a number to choose a class: 1 + The dataset you selected uses the following sample type: + + class CrudeSample(dict): + """Generic sample type to be processed later.""" + + CrudeWebdataset does not need a field map. You will need to provide a `Cooker` for your dataset samples in your `TaskEncoder`. + Furthermore, you might want to add `subflavors` in your meta dataset specification. + +training +-------------------------- + +To launch training on one node + +.. code-block:: bash + + torchrun --nproc-per-node 8 nemo/collections/diffusion/train.py --yes --factory pretrain_xl + +To launch training on multiple nodes using Slurm + +.. code-block:: bash + + sbatch nemo/collections/diffusion/scripts/train.sh --factory pretrain_xl + + +Citations +--------- + +.. [1] William Peebles and Saining Xie, "Scalable Diffusion Models with Transformers," *arXiv preprint arXiv:2212.09748*, 2022. \ No newline at end of file diff --git a/nemo_vfm/diffusion/recipes/__init__.py b/nemo_vfm/diffusion/recipes/__init__.py new file mode 100644 index 00000000..5c6e2391 --- /dev/null +++ b/nemo_vfm/diffusion/recipes/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.diffusion.recipes import flux_535m +from nemo.collections.llm.recipes.log.default import default_log, default_resume +from nemo.collections.llm.recipes.optim import adam, sgd +from nemo.collections.llm.recipes.run.executor import torchrun + + +__all__ = [ + "adam", + "sgd", + "default_log", + "default_resume", + "flux_535m", + "torchrun", +] diff --git a/nemo_vfm/diffusion/recipes/flux_12b.py b/nemo_vfm/diffusion/recipes/flux_12b.py new file mode 100644 index 00000000..0666d747 --- /dev/null +++ b/nemo_vfm/diffusion/recipes/flux_12b.py @@ -0,0 +1,210 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Optional + +import lightning.pytorch as pl +import nemo_run as run +import torch +from lightning.pytorch.callbacks.callback import Callback +from megatron.core.distributed import DistributedDataParallelConfig +from megatron.core.optimizer import OptimizerConfig +from nemo import lightning as nl +from nemo.collections.diffusion.data.diffusion_mock_datamodule import MockDataModule +from nemo.collections.diffusion.models.flux.model import FluxModelParams, MegatronFluxModel +from nemo.collections.llm.api import pretrain +from nemo.collections.llm.recipes.log.default import default_log, tensorboard_logger +from nemo.collections.llm.recipes.precision.mixed_precision import bf16_mixed +from nemo.utils.exp_manager import TimingCallback + + +NAME = "flux_12b" + + +@run.cli.factory +@run.autoconvert +def flux_mock_datamodule() -> pl.LightningDataModule: + """Mock Datamodule Initialization""" + data_module = MockDataModule( + image_h=1024, + image_w=1024, + micro_batch_size=1, + global_batch_size=2, + image_precached=True, + text_precached=True, + ) + return data_module + + +@run.cli.factory(name=NAME) +def model() -> run.Config[pl.LightningModule]: + """ + Factory function to create a FLUX 12B model configuration. + + Returns: + run.Config[pl.LightningModule]: Configuration for the FLUX 12B model. + + """ + return run.Config(MegatronFluxModel, flux_params=run.Config(FluxModelParams)) + + +def trainer( + tensor_parallelism: int = 1, + pipeline_parallelism: int = 1, + pipeline_parallelism_type: Optional[torch.dtype] = None, + virtual_pipeline_parallelism: Optional[int] = None, + context_parallelism: int = 1, + sequence_parallelism: bool = False, + num_nodes: int = 1, + num_gpus_per_node: int = 8, + max_steps: int = 1168251, + callbacks: Optional[list[run.Config[Callback]]] = None, +) -> run.Config[nl.Trainer]: + """ + Configure the NeMo Lightning Trainer for FLUX 12B model. + + This function sets up the distributed training strategy and other training parameters. + + Args: + tensor_parallelism (int): Degree of tensor model parallelism. + pipeline_parallelism (int): Degree of pipeline model parallelism. + pipeline_parallelism_type (Optional[torch.dtype]): Data type for pipeline parallelism. + virtual_pipeline_parallelism (Optional[int]): Size of virtual pipeline parallelism. + context_parallelism (int): Degree of context parallelism. + sequence_parallelism (bool): Whether to use sequence parallelism. + num_nodes (int): Number of compute nodes to use. + num_gpus_per_node (int): Number of GPUs per node. + max_steps (int): Maximum number of training steps. + callbacks (Optional[list[run.Config[Callback]]]): List of callback configurations. + + Returns: + run.Config[nl.Trainer]: Configuration for the NeMo Lightning Trainer. + + Examples: + CLI usage: + $ nemo llm pretrain trainer=flux_12b ... + + Python API usage: + >>> trainer_config = trainer(num_nodes=2, num_gpus_per_node=8) + >>> print(trainer_config) + + Note: + For more information on distributed training strategies, refer to the + NeMo documentation on multi-GPU and multi-node training. + """ + strategy = run.Config( + nl.MegatronStrategy, + tensor_model_parallel_size=tensor_parallelism, + pipeline_model_parallel_size=pipeline_parallelism, + context_parallel_size=context_parallelism, + sequence_parallel=sequence_parallelism, + pipeline_dtype=pipeline_parallelism_type, + gradient_accumulation_fusion=True, + ddp=run.Config( + DistributedDataParallelConfig, + use_custom_fsdp=True, + data_parallel_sharding_strategy="optim_grads_params", + check_for_nan_in_grad=True, + grad_reduce_in_fp32=True, + overlap_grad_reduce=True, + overlap_param_gather=True, + ), + ) + + trainer = run.Config( + nl.Trainer, + accelerator="gpu", + accumulate_grad_batches=1, + callbacks=callbacks, + devices=num_gpus_per_node, + limit_test_batches=50, + limit_val_batches=32, + log_every_n_steps=10, + max_steps=max_steps, + num_nodes=num_nodes, + plugins=bf16_mixed(), + strategy=strategy, + use_distributed_sampler=False, + val_check_interval=2000, + ) + + return trainer + + +@run.cli.factory(target=pretrain, name=NAME) +def pretrain_recipe( + dir: Optional[str] = None, + name: str = "default", + num_nodes: int = 1, + num_gpus_per_node: int = 8, + fn: Callable = pretrain, +) -> run.Partial: + """ + Create a pre-training recipe for FLUX 12B model. + + This function sets up a complete configuration for pre-training, including + model, trainer, data, logging, optimization, and resumption settings. + + Args: + dir (Optional[str]): Directory for saving logs and checkpoints. + name (str): Name of the pre-training run. + num_nodes (int): Number of compute nodes to use. + num_gpus_per_node (int): Number of GPUs per node. + fn (Callable): The pre-training function to use. + + Returns: + run.Partial: Partial configuration for pre-training. + + Examples: + CLI usage: + $ nemo llm pretrain --factory flux_12b + + Python API usage: + >>> recipe = pretrain_recipe(name="flux_12b_pretrain", num_nodes=1) + >>> print(recipe) + + Note: + For more details on pre-training LLMs with NeMo, see the pre-training + guide in the `examples/llm/pretrain/` directory. + """ + recipe = run.Partial( + fn, + model=model(), + trainer=trainer( + num_nodes=num_nodes, + num_gpus_per_node=num_gpus_per_node, + callbacks=[run.Config(TimingCallback)], + ), + data=flux_mock_datamodule(), + log=default_log(dir=dir, name=name, tensorboard_logger=tensorboard_logger(name=name)), + optim=run.Config( + nl.MegatronOptimizerModule, + config=run.Config( + OptimizerConfig, + lr=1e-4, + bf16=True, + use_distributed_optimizer=True, + weight_decay=0, + ), + ), + resume=run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + ), + model_transform=None, + ) + + return recipe diff --git a/nemo_vfm/diffusion/recipes/flux_535m.py b/nemo_vfm/diffusion/recipes/flux_535m.py new file mode 100644 index 00000000..49514256 --- /dev/null +++ b/nemo_vfm/diffusion/recipes/flux_535m.py @@ -0,0 +1,125 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional + +import lightning.pytorch as pl +import nemo_run as run +import torch +from megatron.core.distributed import DistributedDataParallelConfig +from megatron.core.optimizer import OptimizerConfig +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.models.flux.model import FluxModelParams, MegatronFluxModel +from nemo.collections.llm.recipes.log.default import default_resume, tensorboard_logger + + +NAME = "flux-535m" + + +@dataclass +class DummyModelParams(FluxModelParams): + """ + Initialize a toy model that only has one layer of each type. + """ + + def __post_init__(self): + self.t5_params = None + self.clip_params = None + self.vae_config = None + self.flux_config.num_single_layers = 1 + self.flux_config.num_joint_layers = 1 + + +@run.cli.factory(name=NAME) +def model() -> run.Config[pl.LightningModule]: + """ + Factory function to create a Flux sample model configuration with only 1 transformer layers. + + Returns: + run.Config[pl.LightningModule]: Configuration for the Flux sample (535 million) model. + + Examples: + CLI usage: + $ nemo llm pretrain model=bert_110m ... + + Python API usage: + >>> model_config = model(flux_params) + >>> print(model_config) + """ + + return run.Config( + MegatronFluxModel, + flux_params=run.Config( + DummyModelParams, + ), + ) + + +@run.cli.factory(target=llm.train, name=NAME) +def unit_test_recipe( + name: str = "default", + dir: Optional[str] = None, + num_nodes: int = 1, + num_gpus_per_node: int = 8, +): + """ + Flux ci test recipe with default trainer, no parallelism. + """ + return run.Partial( + llm.train, + model=model(), + trainer=run.Config( + nl.Trainer, + devices=num_gpus_per_node, + num_nodes=num_nodes, + accelerator="gpu", + strategy=run.Config( + nl.MegatronStrategy, + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + pipeline_dtype=torch.bfloat16, + ddp=run.Config( + DistributedDataParallelConfig, + check_for_nan_in_grad=True, + grad_reduce_in_fp32=True, + ), + ), + plugins=run.Config(nl.MegatronMixedPrecision, precision="bf16-mixed"), + num_sanity_val_steps=0, + max_steps=10, + log_every_n_steps=1, + ), + log=run.Config( + nl.NeMoLogger, + ckpt=None, + name=name, + tensorboard=tensorboard_logger(name=name), + log_dir=dir, + ), + optim=run.Config( + nl.MegatronOptimizerModule, + config=run.Config( + OptimizerConfig, + lr=1e-4, + bf16=True, + use_distributed_optimizer=True, + weight_decay=0, + ), + ), + resume=default_resume(), + ) diff --git a/nemo_vfm/diffusion/sampler/__init__.py b/nemo_vfm/diffusion/sampler/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/sampler/batch_ops.py b/nemo_vfm/diffusion/sampler/batch_ops.py new file mode 100644 index 00000000..956dfbee --- /dev/null +++ b/nemo_vfm/diffusion/sampler/batch_ops.py @@ -0,0 +1,104 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from torch import Tensor + + +def common_broadcast(x: Tensor, y: Tensor) -> tuple[Tensor, Tensor]: + """ + Broadcasts two tensors to have the same shape by adding singleton dimensions where necessary. + + Args: + x (Tensor): The first input tensor. + y (Tensor): The second input tensor. + + Returns: + tuple[Tensor, Tensor]: A tuple containing the two tensors with broadcasted shapes. + + Raises: + AssertionError: If the dimensions of the tensors do not match at any axis within their common dimensions. + """ + ndims1 = x.ndim + ndims2 = y.ndim + + common_ndims = min(ndims1, ndims2) + for axis in range(common_ndims): + assert x.shape[axis] == y.shape[axis], "Dimensions not equal at axis {}".format(axis) + + if ndims1 < ndims2: + x = x.reshape(x.shape + (1,) * (ndims2 - ndims1)) + elif ndims2 < ndims1: + y = y.reshape(y.shape + (1,) * (ndims1 - ndims2)) + + return x, y + + +def batch_add(x: Tensor, y: Tensor) -> Tensor: + """ + Adds two tensors element-wise after broadcasting them to a common shape. + + Args: + x (Tensor): The first input tensor. + y (Tensor): The second input tensor. + + Returns: + Tensor: The element-wise sum of the input tensors after broadcasting. + """ + x, y = common_broadcast(x, y) + return x + y + + +def batch_mul(x: Tensor, y: Tensor) -> Tensor: + """ + Multiplies two tensors element-wise after broadcasting them to a common shape. + + Args: + x (Tensor): The first input tensor. + y (Tensor): The second input tensor. + + Returns: + Tensor: The element-wise product of the input tensors after broadcasting. + """ + x, y = common_broadcast(x, y) + return x * y + + +def batch_sub(x: Tensor, y: Tensor) -> Tensor: + """ + Subtracts two tensors element-wise after broadcasting them to a common shape. + + Args: + x (Tensor): The first input tensor. + y (Tensor): The second input tensor. + + Returns: + Tensor: The result of element-wise subtraction of the input tensors. + """ + x, y = common_broadcast(x, y) + return x - y + + +def batch_div(x: Tensor, y: Tensor) -> Tensor: + """ + Divides two tensors element-wise after broadcasting them to a common shape. + + Args: + x (Tensor): The first input tensor. + y (Tensor): The second input tensor. + + Returns: + Tensor: The result of element-wise division of `x` by `y` after broadcasting. + """ + x, y = common_broadcast(x, y) + return x / y diff --git a/nemo_vfm/diffusion/sampler/conditioner.py b/nemo_vfm/diffusion/sampler/conditioner.py new file mode 100755 index 00000000..05c6f233 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/conditioner.py @@ -0,0 +1,757 @@ +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. + +# pylint: disable=C0115,C0116,C0301 + +import copy +from abc import ABC, abstractmethod +from collections import defaultdict +from contextlib import nullcontext +from dataclasses import dataclass, fields +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from einops import reduce +from einops.layers.torch import Rearrange +from nemo.collections.diffusion.sampler.batch_ops import batch_mul +from torch import nn + + +# Utils + + +def count_params(model, verbose=False): + total_params = sum(p.numel() for p in model.parameters()) + if verbose: + print(f"{model.__class__.__name__} has {total_params * 1.0e-6:.2f} M params.") + return total_params + + +def disabled_train(self: Any, mode: bool = True) -> Any: + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +# TODO: Implement in MCore later +class FourierFeatures(nn.Module): + """ + Implements a layer that generates Fourier features from input tensors, based on randomly sampled + frequencies and phases. This can help in learning high-frequency functions in low-dimensional problems. + + [B] -> [B, D] + + Parameters: + num_channels (int): The number of Fourier features to generate. + bandwidth (float, optional): The scaling factor for the frequency of the Fourier features. Defaults to 1. + normalize (bool, optional): If set to True, the outputs are scaled by sqrt(2), usually to normalize + the variance of the features. Defaults to False. + + Example: + >>> layer = FourierFeatures(num_channels=256, bandwidth=0.5, normalize=True) + >>> x = torch.randn(10, 256) # Example input tensor + >>> output = layer(x) + >>> print(output.shape) # Expected shape: (10, 256) + """ + + def __init__(self, num_channels, bandwidth=1, normalize=False): + super().__init__() + self.register_buffer("freqs", 2 * np.pi * bandwidth * torch.randn(num_channels), persistent=True) + self.register_buffer("phases", 2 * np.pi * torch.rand(num_channels), persistent=True) + self.gain = np.sqrt(2) if normalize else 1 + + def forward(self, x, gain: float = 1.0): + """ + Apply the Fourier feature transformation to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + gain (float, optional): An additional gain factor applied during the forward pass. Defaults to 1. + + Returns: + torch.Tensor: The transformed tensor, with Fourier features applied. + """ + in_dtype = x.dtype + x = x.to(torch.float32).ger(self.freqs.to(torch.float32)).add(self.phases.to(torch.float32)) + x = x.cos().mul(self.gain * gain).to(in_dtype) + return x + + +# TODO: Switch to MCore implementation later + +# ---------------------- Feed Forward Network ----------------------- + + +class FeedForward(nn.Module): + """ + Transformer FFN with optional gating + + Parameters: + d_model (int): Dimensionality of input features. + d_ff (int): Dimensionality of the hidden layer. + dropout (float, optional): Dropout rate applied after the activation function. Defaults to 0.1. + activation (callable, optional): The activation function applied after the first linear layer. + Defaults to nn.ReLU(). + is_gated (bool, optional): If set to True, incorporates gating mechanism to the feed-forward layer. + Defaults to False. + bias (bool, optional): If set to True, adds a bias to the linear layers. Defaults to True. + + Example: + >>> ff = FeedForward(d_model=512, d_ff=2048) + >>> x = torch.randn(64, 10, 512) # Example input tensor + >>> output = ff(x) + >>> print(output.shape) # Expected shape: (64, 10, 512) + """ + + def __init__( + self, + d_model: int, + d_ff: int, + dropout: float = 0.1, + activation=nn.ReLU(), + is_gated: bool = False, + bias: bool = False, + ): + super().__init__() + self.layer1 = nn.Linear(d_model, d_ff, bias=bias) + self.layer2 = nn.Linear(d_ff, d_model, bias=bias) + self.dropout = nn.Dropout(dropout) + self.activation = activation + self.is_gated = is_gated + if is_gated: + self.linear_gate = nn.Linear(d_model, d_ff, bias=False) + + def forward(self, x: torch.Tensor): + g = self.activation(self.layer1(x)) + if self.is_gated: + x = g * self.linear_gate(x) + else: + x = g + # Apply dropout + x = self.dropout(x) + return self.layer2(x) + + +class SwiGLUFeedForward(FeedForward): + def __init__(self, d_model: int, d_ff: int, dropout: float = 0.1): + super().__init__( + d_model=d_model, + d_ff=d_ff, + dropout=dropout, + activation=nn.SiLU(), + is_gated=True, + bias=False, + ) + + +class AbstractEmbModel(nn.Module): + def __init__(self): + super().__init__() + + self._is_trainable = None + self._dropout_rate = None + self._input_key = None + # TODO: (qsh 2024-02-14) a cleaner define or we use return dict by default? + self._return_dict = False + + @property + def is_trainable(self) -> bool: + return self._is_trainable + + @property + def dropout_rate(self) -> Union[float, torch.Tensor]: + return self._dropout_rate + + @property + def input_key(self) -> str: + return self._input_key + + @property + def is_return_dict(self) -> bool: + return self._return_dict + + @is_trainable.setter + def is_trainable(self, value: bool): + self._is_trainable = value + + @dropout_rate.setter + def dropout_rate(self, value: Union[float, torch.Tensor]): + self._dropout_rate = value + + @input_key.setter + def input_key(self, value: str): + self._input_key = value + + @is_return_dict.setter + def is_return_dict(self, value: bool): + self._return_dict = value + + @is_trainable.deleter + def is_trainable(self): + del self._is_trainable + + @dropout_rate.deleter + def dropout_rate(self): + del self._dropout_rate + + @input_key.deleter + def input_key(self): + del self._input_key + + @is_return_dict.deleter + def is_return_dict(self): + del self._return_dict + + def random_dropout_input( + self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None, key: Optional[str] = None + ) -> torch.Tensor: + del key + dropout_rate = dropout_rate if dropout_rate is not None else self.dropout_rate + return batch_mul( + torch.bernoulli((1.0 - dropout_rate) * torch.ones(in_tensor.shape[0])).type_as(in_tensor), + in_tensor, + ) + + def details(self) -> str: + return "" + + def summary(self) -> str: + input_key = self.input_key if self.input_key is not None else getattr(self, "input_keys", None) + return ( + f"{self.__class__.__name__} \n\tinput key: {input_key}" + f"\n\tParam count: {count_params(self, False)} \n\tTrainable: {self.is_trainable}" + f"\n\tDropout rate: {self.dropout_rate}" + f"\n\t{self.details()}" + ) + + +class TrainingOnlyEmbModel(AbstractEmbModel): + """ + Class to denote special case embedding that is + only used for training, and is dropped out at inference + """ + + def __init__(self): + super().__init__() + + +class ReMapkey(AbstractEmbModel): + def __init__(self, output_key: Optional[str] = None, dtype: Optional[str] = None): + super().__init__() + self.output_key = output_key + self.dtype = { + None: None, + "float": torch.float32, + "bfloat16": torch.bfloat16, + "half": torch.float16, + "float16": torch.float16, + "int": torch.int32, + "long": torch.int64, + }[dtype] + + def forward(self, element: torch.Tensor) -> Dict[str, torch.Tensor]: + key = self.output_key if self.output_key else self.input_key + if isinstance(element, torch.Tensor): + element = element.to(dtype=self.dtype) + return {key: element} + + def details(self) -> str: + key = self.output_key if self.output_key else self.input_key + return f"Output key: {key} \n\tDtype: {self.dtype}" + + +class ScalarEmb(AbstractEmbModel): + def __init__( + self, + num_channels: int, + num_token_per_scalar_condition: int = 2, + num_scalar_condition: int = 4, + output_key: Optional[str] = None, + ): + super().__init__() + self.model_channels = num_channels + self.num_token_per_scalar_condition = num_token_per_scalar_condition + self.num_scalar_condition = num_scalar_condition + self.output_key = output_key + self.feature_proj = nn.Sequential( + FourierFeatures(num_channels * num_token_per_scalar_condition), + Rearrange("b (l c) -> b l c", l=num_token_per_scalar_condition), + nn.LayerNorm(num_channels), + SwiGLUFeedForward(num_channels, num_channels, 0.0), + Rearrange("(b n) l c -> b (n l) c", n=num_scalar_condition), + ) + + def forward(self, element: torch.Tensor) -> Dict[str, torch.Tensor]: + batch_size = element.shape[0] + assert torch.numel(element) == batch_size * self.num_scalar_condition, ( + f"element shape {element.shape} does not match with {batch_size}x{self.num_scalar_condition}" + ) + key = self.output_key if self.output_key else self.input_key + return {key: self.feature_proj(element.flatten())} + + def details(self) -> str: + key = self.output_key if self.output_key else self.input_key + return "\n\t".join( + [ + f"Output key: {key}", + f"num_token_per_scalar_condition: {self.num_token_per_scalar_condition}", + f"num_scalar_condition: {self.num_scalar_condition}", + f"model_channels: {self.model_channels}", + ] + ) + + +class CameraAttr(AbstractEmbModel): + def __init__(self, context_dim: int, num_pitch: int, num_shot_type: int, num_depth_of_field: int): + super().__init__() + self.num_pitch = num_pitch + self.num_shot_type = num_shot_type + self.num_depth_of_field = num_depth_of_field + + self.pitch_projection = nn.Embedding(self.num_pitch, context_dim) + self.shot_type_projection = nn.Embedding(self.num_shot_type, context_dim) + self.depth_of_field_projection = nn.Embedding(self.num_depth_of_field, context_dim) + + def forward(self, camera_attributes: torch.Tensor) -> Dict[str, torch.Tensor]: + pitch_emb = self.pitch_projection(camera_attributes[:, 0].unsqueeze(-1).long()) + shot_type_emb = self.shot_type_projection(camera_attributes[:, 1].unsqueeze(-1).long()) + depth_of_field_emb = self.depth_of_field_projection(camera_attributes[:, 2].unsqueeze(-1).long()) + + tokens = torch.cat([pitch_emb, shot_type_emb, depth_of_field_emb], dim=1) + + return { + "crossattn_emb": tokens, + "crossattn_mask": torch.ones(tokens.shape[0], tokens.shape[1], device=tokens.device, dtype=torch.bool), + } + + def details(self) -> str: + return f"Num pitch: {self.num_pitch} \n\tNum shot type: {self.num_shot_type} \n\tNum depth of field: {self.num_depth_of_field} \n\tOutput key: [crossattn_emb, crossattn_mask]" + + +class HumanAttr(AbstractEmbModel): + def __init__(self, context_dim, num_race, num_gender, num_age, max_num_humans): + super().__init__() + self.num_race = num_race + self.num_gender = num_gender + self.num_age = num_age + self.max_num_humans = max_num_humans + + self.race_projection = nn.Embedding(self.num_race, context_dim) + self.gender_projection = nn.Embedding(self.num_gender, context_dim) + self.age_projection = nn.Embedding(self.num_age, context_dim) + self.num_human_projection = nn.Embedding(self.max_num_humans + 1, context_dim) + + self.human_projection = nn.Linear(3 * context_dim, context_dim, bias=True) + + self.human_bias = nn.Parameter(torch.zeros(5, context_dim)) + + def forward(self, human_attributes: torch.Tensor) -> Dict[str, torch.Tensor]: + num_humans = human_attributes.max(dim=2)[0].bool().sum(dim=1) + num_human_emb = self.num_human_projection(num_humans.unsqueeze(-1).long()) + + race_emb = self.race_projection(human_attributes[:, :, 0].long()) + gender_emb = self.gender_projection(human_attributes[:, :, 1].long()) + age_emb = self.age_projection(human_attributes[:, :, 2].long()) + + # TODO: (qsh 2024-02-14) I do not understand the purpose of additional linear layer instead of just concatenation + human_emb = self.human_projection( + torch.cat([race_emb, gender_emb, age_emb], dim=-1) + ) + self.human_bias.unsqueeze(0) + + token = torch.cat([human_emb, num_human_emb], dim=1) + + return { + "crossattn_emb": token, + "crossattn_mask": torch.ones(token.shape[0], token.shape[1], device=token.device, dtype=torch.bool), + } + + def details(self) -> str: + return f"NumRace : {self.num_race} \n\tNumGender : {self.num_gender} \n\tNumAge : {self.num_age} \n\tMaxNumHumans : {self.max_num_humans} \n\tOutput key: [crossattn_emb, crossattn_mask]" + + +class PaddingMask(AbstractEmbModel): + def __init__(self, spatial_reduction: int = 16): + super().__init__() + self.spatial_reduction = spatial_reduction + + def forward(self, mask: torch.Tensor) -> Dict[str, torch.Tensor]: + return { + "padding_mask": reduce( + mask, "... (h n) (w m) -> ... h w", "mean", n=self.spatial_reduction, m=self.spatial_reduction + ) + } + + def details(self) -> str: + return f"Spatial reduction: {self.spatial_reduction} \n\tOutput key: padding_mask" + + +class SingleAttr(AbstractEmbModel): + def __init__(self, context_dim: int, num_label: int): + super().__init__() + self.num_label = num_label + self.emb = nn.Embedding(num_label, context_dim) + + def forward(self, attr: torch.Tensor) -> Dict[str, torch.Tensor]: + token = self.emb(attr.long()) + return { + "crossattn_emb": token, + "crossattn_mask": torch.ones(attr.shape[0], 1, device=attr.device, dtype=torch.bool), + } + + def details(self) -> str: + return f"NumLabel : {self.num_label} \n\tOutput key: [crossattn_emb, crossattn_mask]" + + +class TextAttr(AbstractEmbModel): + def __init__(self): + super().__init__() + + def forward(self, token: torch.Tensor, mask: torch.Tensor): + return {"crossattn_emb": token, "crossattn_mask": mask} + + def random_dropout_input( + self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None, key: Optional[str] = None + ) -> torch.Tensor: + if key is not None and "mask" in key: + return in_tensor + return super().random_dropout_input(in_tensor, dropout_rate, key) + + def details(self) -> str: + return "Output key: [crossattn_emb, crossattn_mask]" + + +class BooleanFlag(AbstractEmbModel): + def __init__(self, output_key: Optional[str] = None): + super().__init__() + self.output_key = output_key + + def forward(self, *args, **kwargs) -> Dict[str, torch.Tensor]: + del args, kwargs + key = self.output_key if self.output_key else self.input_key + return {key: self.flag} + + def random_dropout_input( + self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None, key: Optional[str] = None + ) -> torch.Tensor: + del key + dropout_rate = dropout_rate if dropout_rate is not None else self.dropout_rate + self.flag = torch.bernoulli((1.0 - dropout_rate) * torch.ones(1)).bool().to(device=in_tensor.device) + return in_tensor + + def details(self) -> str: + key = self.output_key if self.output_key else self.input_key + return f"Output key: {key} \n\t This is a boolean flag" + + +class GeneralConditioner(nn.Module, ABC): + """ + An abstract module designed to handle various embedding models with conditional and unconditional configurations. + This abstract base class initializes and manages a collection of embedders that can dynamically adjust + their dropout rates based on conditioning. + + Attributes: + KEY2DIM (dict): A mapping from output keys to dimensions used for concatenation. + embedders (nn.ModuleDict): A dictionary containing all embedded models initialized and configured + based on the provided configurations. + + Parameters: + emb_models (Union[List, Any]): A dictionary where keys are embedder names and values are configurations + for initializing the embedders. + + Example: + See Edify4ConditionerConfig + """ + + KEY2DIM = {"crossattn_emb": 1, "crossattn_mask": 1} + + def __init__(self, **emb_models: Union[List, Any]): + super().__init__() + self.embedders = nn.ModuleDict() + for n, (emb_name, embconfig) in enumerate(emb_models.items()): + embedder = embconfig.obj + assert isinstance(embedder, AbstractEmbModel), ( + f"embedder model {embedder.__class__.__name__} has to inherit from AbstractEmbModel" + ) + embedder.is_trainable = getattr(embconfig, "is_trainable", True) + embedder.dropout_rate = getattr(embconfig, "dropout_rate", 0.0) + if not embedder.is_trainable: + embedder.train = disabled_train + for param in embedder.parameters(): + param.requires_grad = False + embedder.eval() + + if hasattr(embconfig, "input_key"): + embedder.input_key = embconfig.input_key + elif hasattr(embconfig, "input_keys"): + embedder.input_keys = embconfig.input_keys + else: + raise KeyError(f"need either 'input_key' or 'input_keys' for embedder {embedder.__class__.__name__}") + + self.embedders[emb_name] = embedder + + @abstractmethod + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> Any: + """Should be implemented in subclasses to handle conditon datatype""" + raise NotImplementedError + + def _forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> Dict: + """ + Processes the input batch through all configured embedders, applying conditional dropout rates if specified. + Output tensors for each key are concatenated along the dimensions specified in KEY2DIM. + + Parameters: + batch (Dict): The input data batch to process. + override_dropout_rate (Optional[Dict[str, float]]): Optional dictionary to override default dropout rates + per embedder key. + + Returns: + Dict: A dictionary of output tensors concatenated by specified dimensions. + + Note: + In case the network code is sensitive to the order of concatenation, you can either control the order via \ + config file or make sure the embedders return a unique key for each output. + """ + output = defaultdict(list) + if override_dropout_rate is None: + override_dropout_rate = {} + + # make sure emb_name in override_dropout_rate is valid + for emb_name in override_dropout_rate.keys(): + assert emb_name in self.embedders, f"invalid name found {emb_name}" + + for emb_name, embedder in self.embedders.items(): + embedding_context = nullcontext if embedder.is_trainable else torch.no_grad + with embedding_context(): + if hasattr(embedder, "input_key") and (embedder.input_key is not None): + emb_out = embedder( + embedder.random_dropout_input( + batch[embedder.input_key], override_dropout_rate.get(emb_name, None) + ) + ) + elif hasattr(embedder, "input_keys"): + emb_out = embedder( + *[ + embedder.random_dropout_input(batch[k], override_dropout_rate.get(emb_name, None), k) + for k in embedder.input_keys + ] + ) + for k, v in emb_out.items(): + output[k].append(v) + # Concatenate the outputs + return {k: torch.cat(v, dim=self.KEY2DIM.get(k, -1)) for k, v in output.items()} + + def get_condition_uncondition( + self, + data_batch: Dict, + ) -> Tuple[Any, Any]: + """ + Processes the provided data batch to generate two sets of outputs: conditioned and unconditioned. This method + manipulates the dropout rates of embedders to simulate two scenarios — one where all conditions are applied + (conditioned), and one where they are removed or reduced to the minimum (unconditioned). + + This method first sets the dropout rates to zero for the conditioned scenario to fully apply the embedders' effects. + For the unconditioned scenario, it sets the dropout rates to 1 (or to 0 if the initial unconditional dropout rate + is insignificant) to minimize the embedders' influences, simulating an unconditioned generation. + + Parameters: + data_batch (Dict): The input data batch that contains all necessary information for embedding processing. The + data is expected to match the required format and keys expected by the embedders. + + Returns: + Tuple[Any, Any]: A tuple containing two condition: + - The first one contains the outputs with all embedders fully applied (conditioned outputs). + - The second one contains the outputs with embedders minimized or not applied (unconditioned outputs). + """ + cond_dropout_rates, dropout_rates = {}, {} + for emb_name, embedder in self.embedders.items(): + cond_dropout_rates[emb_name] = 0.0 + dropout_rates[emb_name] = 1.0 if embedder.dropout_rate > 1e-4 else 0.0 + + condition: Any = self(data_batch, override_dropout_rate=cond_dropout_rates) + un_condition: Any = self(data_batch, override_dropout_rate=dropout_rates) + return condition, un_condition + + def get_condition_with_negative_prompt( + self, + data_batch: Dict, + ) -> Tuple[Any, Any]: + """ + Similar functionality as get_condition_uncondition + But use negative prompts for unconditon + """ + cond_dropout_rates, uncond_dropout_rates = {}, {} + for emb_name, embedder in self.embedders.items(): + cond_dropout_rates[emb_name] = 0.0 + if isinstance(embedder, TextAttr): + uncond_dropout_rates[emb_name] = 0.0 + else: + uncond_dropout_rates[emb_name] = 1.0 if embedder.dropout_rate > 1e-4 else 0.0 + + data_batch_neg_prompt = copy.deepcopy(data_batch) + if "neg_t5_text_embeddings" in data_batch_neg_prompt: + if isinstance(data_batch_neg_prompt["neg_t5_text_embeddings"], torch.Tensor): + data_batch_neg_prompt["t5_text_embeddings"] = data_batch_neg_prompt["neg_t5_text_embeddings"] + data_batch_neg_prompt["t5_text_mask"] = data_batch_neg_prompt["neg_t5_text_mask"] + + condition: Any = self(data_batch, override_dropout_rate=cond_dropout_rates) + un_condition: Any = self(data_batch_neg_prompt, override_dropout_rate=uncond_dropout_rates) + + return condition, un_condition + + +@dataclass +class Edify4Condition: + crossattn_emb: torch.Tensor + crossattn_mask: torch.Tensor + padding_mask: Optional[torch.Tensor] = None + scalar_feature: Optional[torch.Tensor] = None + pos_ids: Optional[torch.Tensor] = None + + def to_dict(self) -> Dict[str, Optional[torch.Tensor]]: + return {f.name: getattr(self, f.name) for f in fields(self)} + + +class Edify4Conditioner(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> Edify4Condition: + output = super()._forward(batch, override_dropout_rate) + return Edify4Condition(**output) + + +class DataType(Enum): + IMAGE = "image" + VIDEO = "video" + MIX = "mix" + + +@dataclass +class BaseVideoCondition: + crossattn_emb: torch.Tensor + crossattn_mask: torch.Tensor + data_type: DataType = DataType.VIDEO + padding_mask: Optional[torch.Tensor] = None + fps: Optional[torch.Tensor] = None + num_frames: Optional[torch.Tensor] = None + image_size: Optional[torch.Tensor] = None + scalar_feature: Optional[torch.Tensor] = None + trajectory: Optional[torch.Tensor] = None + frame_repeat: Optional[torch.Tensor] = None + + def to_dict(self) -> Dict[str, Optional[torch.Tensor]]: + return {f.name: getattr(self, f.name) for f in fields(self)} + + +class VideoConditioner(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> BaseVideoCondition: + output = super()._forward(batch, override_dropout_rate) + return BaseVideoCondition(**output) + + +@dataclass +class VideoExtendCondition(BaseVideoCondition): + video_cond_bool: Optional[torch.Tensor] = None # whether or not it conditioned on video + gt_latent: Optional[torch.Tensor] = None + condition_video_indicator: Optional[torch.Tensor] = None # 1 for condition region + action_control_condition: Optional[torch.Tensor] = ( + None # Optional action control embedding input to the V2W model for fine-tuning. + ) + + # condition_video_input_mask will concat to the input of network, along channel dim; + # Will be concat with the input tensor + condition_video_input_mask: Optional[torch.Tensor] = None + # condition_video_augment_sigma: (B, T) tensor of sigma value for the conditional input augmentation, only valid when apply_corruption_to_condition_region is "noise_with_sigma" or "noise_with_sigma_fixed" + condition_video_augment_sigma: Optional[torch.Tensor] = None + # pose conditional input, will be concat with the input tensor + condition_video_pose: Optional[torch.Tensor] = None + + # NOTE(jjennings): All members below can be wrapped into a separate "Config" class + + dropout_rate: float = 0.2 + input_key: str = "fps" # This is a placeholder, we never use this value + # Config below are for long video generation only + compute_loss_for_condition_region: bool = False # Compute loss for condition region + + # How to sample condition region during training. "first_random_n" set the first n frames to be condition region, n is random, "random" set the condition region to be random, + condition_location: str = "first_n" + random_conditon_rate: float = 0.5 # The rate to sample the condition region randomly + first_random_n_num_condition_t_max: int = 4 # The maximum number of frames to sample as condition region, used when condition_location is "first_random_n" + first_random_n_num_condition_t_min: int = 0 # The minimum number of frames to sample as condition region, used when condition_location is "first_random_n" + + # How to dropout value of the conditional input frames + cfg_unconditional_type: str = "zero_condition_region_condition_mask" # Unconditional type. "zero_condition_region_condition_mask" set the input to zero for condition region, "noise_x_condition_region" set the input to x_t, same as the base model + + # How to corrupt the condition region + apply_corruption_to_condition_region: str = "noise_with_sigma_fixed" # Apply corruption to condition region, option: "gaussian_blur", "noise_with_sigma", "clean" (inference), "noise_with_sigma_fixed" (inference) + # Inference only option: list of sigma value for the corruption at different chunk id, used when apply_corruption_to_condition_region is "noise_with_sigma" or "noise_with_sigma_fixed" + # apply_corruption_to_condition_region_sigma_value: [float] = [0.001, 0.2] + [ + # 0.5 + # ] * 10 # Sigma value for the corruption, used when apply_corruption_to_condition_region is "noise_with_sigma_fixed" + + # Add augment_sigma condition to the network + condition_on_augment_sigma: bool = False + # The following arguments is to match with previous implementation where we use train sde to sample augment sigma (with adjust video noise turn on) + augment_sigma_sample_p_mean: float = 0.0 # Mean of the augment sigma + augment_sigma_sample_p_std: float = 1.0 # Std of the augment sigma + augment_sigma_sample_multiplier: float = 4.0 # Multipler of augment sigma + + # Add pose condition to the network + add_pose_condition: bool = False + + # Sample PPP... from IPPP... sequence + sample_tokens_start_from_p_or_i: bool = False + + # Normalize the input condition latent + normalize_condition_latent: bool = False + + +class VideoExtendConditioner(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> VideoExtendCondition: + output = super()._forward(batch, override_dropout_rate) + return VideoExtendCondition(**output) + + +@dataclass +class VideoExtendConditionControl(VideoExtendCondition): + control_input_canny: Optional[torch.Tensor] = None + control_input_blur: Optional[torch.Tensor] = None + control_input_canny_blur: Optional[torch.Tensor] = None + control_input_depth: Optional[torch.Tensor] = None + control_input_segmentation: Optional[torch.Tensor] = None + control_input_depth_segmentation: Optional[torch.Tensor] = None + control_input_mask: Optional[torch.Tensor] = None + control_input_human_kpts: Optional[torch.Tensor] = None + control_input_upscale: Optional[torch.Tensor] = None + control_input_identity: Optional[torch.Tensor] = None + base_model: Optional[torch.nn.Module] = None + hint_key: Optional[str] = None + control_weight: Optional[float] = 1.0 + + +class VideoExtendConditionerControl(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> VideoExtendCondition: + output = super()._forward(batch, override_dropout_rate) + return VideoExtendConditionControl(**output) diff --git a/nemo_vfm/diffusion/sampler/conditioner_configs.py b/nemo_vfm/diffusion/sampler/conditioner_configs.py new file mode 100755 index 00000000..15c02f58 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/conditioner_configs.py @@ -0,0 +1,157 @@ +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. + +# pylint: disable=C0115,C0301 + +from typing import Any, List + +import attrs +from nemo.collections.diffusion.sampler.conditioner import ( + BooleanFlag, + ReMapkey, + TextAttr, + VideoConditioner, + VideoExtendConditioner, +) + + +@attrs.define(slots=False) +class TextConfig: + obj: Any = TextAttr() # No arguments + dropout_rate: float = 0.2 + input_keys: List[str] = attrs.field(factory=lambda: ["t5_text_embeddings", "t5_text_mask"]) + + +@attrs.define(slots=False) +class ActionControlConfig: + """ + Action control configuration for V2W model conditioning. + """ + + # default embedding model for action control for reference/readability, overwrite using DiT model config! + obj: Any = ReMapkey(output_key="action_control_condition", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "action" + + +@attrs.define(slots=False) +class FPSConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `fps`. + """ + + obj: Any = ReMapkey(output_key="fps", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "fps" + + +@attrs.define(slots=False) +class PaddingMaskConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `fps`. + """ + + obj: Any = ReMapkey(output_key="padding_mask", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "padding_mask" + + +@attrs.define(slots=False) +class ImageSizeConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `fps`. + """ + + obj: Any = ReMapkey(output_key="image_size", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "image_size" + + +@attrs.define(slots=False) +class NumFramesConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `num_frames`. + """ + + obj: Any = ReMapkey(output_key="num_frames", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "num_frames" + + +@attrs.define(slots=False) +class VideoCondBoolConfig: + obj: Any = BooleanFlag(output_key="video_cond_bool") + + dropout_rate: float = 0.0 + input_key: str = "fps" # This is a placeholder, we never use this value + # Config below are for long video generation only + compute_loss_for_condition_region: bool = False # Compute loss for condition region + + # How to sample condition region during training. "first_random_n" set the first n frames to be condition region, n is random, "random" set the condition region to be random, + condition_location: str = "first_n" + random_conditon_rate: float = 0.5 # The rate to sample the condition region randomly + first_random_n_num_condition_t_max: int = 4 # The maximum number of frames to sample as condition region, used when condition_location is "first_random_n" + first_random_n_num_condition_t_min: int = 0 # The minimum number of frames to sample as condition region, used when condition_location is "first_random_n" + + # How to dropout value of the conditional input frames + cfg_unconditional_type: str = "zero_condition_region_condition_mask" # Unconditional type. "zero_condition_region_condition_mask" set the input to zero for condition region, "noise_x_condition_region" set the input to x_t, same as the base model + + # How to corrupt the condition region + apply_corruption_to_condition_region: str = "noise_with_sigma_fixed" # Apply corruption to condition region, option: "gaussian_blur", "noise_with_sigma", "clean" (inference), "noise_with_sigma_fixed" (inference) + # Inference only option: list of sigma value for the corruption at different chunk id, used when apply_corruption_to_condition_region is "noise_with_sigma" or "noise_with_sigma_fixed" + apply_corruption_to_condition_region_sigma_value: list[float] = ( + [0.001, 0.2] + [0.5] * 10 + ) # Sigma value for the corruption, used when apply_corruption_to_condition_region is "noise_with_sigma_fixed" + + # Add augment_sigma condition to the network + condition_on_augment_sigma: bool = False + # The following arguments is to match with previous implementation where we use train sde to sample augment sigma (with adjust video noise turn on) + augment_sigma_sample_p_mean: float = 0.0 # Mean of the augment sigma + augment_sigma_sample_p_std: float = 1.0 # Std of the augment sigma + augment_sigma_sample_multiplier: float = 4.0 # Multipler of augment sigma + + # Add pose condition to the network + add_pose_condition: bool = False + + # Sample PPP... from IPPP... sequence + sample_tokens_start_from_p_or_i: bool = False + + # Normalize the input condition latent + normalize_condition_latent: bool = False + + +BaseVideoConditionerConfig: Any = VideoConditioner( + text=TextConfig(), +) + +VideoConditionerFPSConfig: Any = VideoConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), +) + +VideoConditionerFpsSizePaddingConfig: Any = VideoConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), +) + +VideoExtendConditionerConfig: Any = VideoExtendConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), +) + +VideoActionExtendConditionerConfig: Any = VideoExtendConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), + action_ctrl=ActionControlConfig(), +) diff --git a/nemo_vfm/diffusion/sampler/context_parallel.py b/nemo_vfm/diffusion/sampler/context_parallel.py new file mode 100644 index 00000000..71906fc4 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/context_parallel.py @@ -0,0 +1,82 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import Tensor +from torch.distributed import ProcessGroup, all_gather, get_process_group_ranks, get_world_size + + +def split_inputs_cp(x: Tensor, seq_dim: int, cp_group: ProcessGroup) -> Tensor: + """ + Split input tensor along the sequence dimension for checkpoint parallelism. + + This function divides the input tensor into equal parts along the specified + sequence dimension, based on the number of ranks in the checkpoint parallelism group. + It then selects the part corresponding to the current rank. + + Args: + x: Input tensor to be split. + seq_dim: The dimension along which to split the input (sequence dimension). + cp_group: The process group for checkpoint parallelism. + + Returns: + A slice of the input tensor corresponding to the current rank. + + Raises: + AssertionError: If the sequence dimension is not divisible by the number of ranks. + """ + cp_ranks = get_process_group_ranks(cp_group) + cp_size = len(cp_ranks) + + assert x.shape[seq_dim] % cp_size == 0, f"{x.shape[seq_dim]} cannot divide cp_size {cp_size}" + x = x.view(*x.shape[:seq_dim], cp_size, x.shape[seq_dim] // cp_size, *x.shape[(seq_dim + 1) :]) + seq_idx = torch.tensor([cp_group.rank()], device=x.device) + x = x.index_select(seq_dim, seq_idx) + # Note that the new sequence length is the original sequence length / cp_size + x = x.view(*x.shape[:seq_dim], -1, *x.shape[(seq_dim + 2) :]) + return x + + +def cat_outputs_cp(x: Tensor, seq_dim: int, cp_group: ProcessGroup) -> Tensor: + """ + Concatenates tensors from multiple processes along a specified dimension. + + This function gathers tensors from all processes in the given process group + and concatenates them along the specified dimension. + + Args: + x (Tensor): The input tensor to be gathered and concatenated. + seq_dim (int): The dimension along which to concatenate the gathered tensors. + cp_group (ProcessGroup): The process group containing all the processes involved in the gathering. + + Returns: + Tensor: A tensor resulting from the concatenation of tensors from all processes. + + Raises: + RuntimeError: If the gathering of tensors fails. + """ + # Number of processes in the group + world_size = get_world_size(cp_group) + + # List to hold tensors from each rank + gathered_tensors = [torch.zeros_like(x) for _ in range(world_size)] + + # Attempt to gather tensors from all ranks + try: + all_gather(gathered_tensors, x, group=cp_group) + except RuntimeError as e: + raise RuntimeError(f"Gathering failed: {e}") + + # Concatenate tensors along the specified dimension + return torch.cat(gathered_tensors, dim=seq_dim) diff --git a/nemo_vfm/diffusion/sampler/cosmos/__init__.py b/nemo_vfm/diffusion/sampler/cosmos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/diffusion/sampler/cosmos/cosmos_control_diffusion_pipeline.py b/nemo_vfm/diffusion/sampler/cosmos/cosmos_control_diffusion_pipeline.py new file mode 100644 index 00000000..1c8f6f3b --- /dev/null +++ b/nemo_vfm/diffusion/sampler/cosmos/cosmos_control_diffusion_pipeline.py @@ -0,0 +1,199 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0301 + + +from typing import Callable, Dict, Tuple, Union + +import numpy as np +import torch +import torch.distributed +from megatron.core import parallel_state +from nemo.collections.diffusion.sampler.context_parallel import cat_outputs_cp, split_inputs_cp +from nemo.collections.diffusion.sampler.cosmos.cosmos_extended_diffusion_pipeline import ExtendedDiffusionPipeline +from nemo.collections.diffusion.sampler.res.res_sampler import COMMON_SOLVER_OPTIONS +from torch import Tensor + + +class CosmosControlDiffusionPipeline(ExtendedDiffusionPipeline): + def __init__(self, *args, **kwargs): + base_model = kwargs.pop("base_model", None) + super().__init__(*args, **kwargs) + self.base_model = base_model + self.net.sigma_data = self.sigma_data + + def get_x0_fn_from_batch_with_condition_latent( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + condition_latent: torch.Tensor = None, + num_condition_t: Union[int, None] = None, + condition_video_augment_sigma_in_inference: float = None, + ) -> Callable: + """ + Generates a callable function `x0_fn` based on the provided data batch and guidance factor. + + This function first processes the input data batch through a conditioning workflow (`conditioner`) to obtain conditioned and unconditioned states. It then defines a nested function `x0_fn` which applies a denoising operation on an input `noise_x` at a given noise level `sigma` using both the conditioned and unconditioned states. + + Args: + - data_batch (Dict): A batch of data used for conditioning. The format and content of this dictionary should align with the expectations of the `self.conditioner` + - guidance (float, optional): A scalar value that modulates the influence of the conditioned state relative to the unconditioned state in the output. Defaults to 1.5. + - is_negative_prompt (bool): use negative prompt t5 in uncondition if true + condition_latent (torch.Tensor): latent tensor in shape B,C,T,H,W as condition to generate video. + - num_condition_t (int): number of condition latent T, used in inference to decide the condition region and config.conditioner.video_cond_bool.condition_location == "first_n" + - condition_video_augment_sigma_in_inference (float): sigma for condition video augmentation in inference + + Returns: + - Callable: A function `x0_fn(noise_x, sigma)` that takes two arguments, `noise_x` and `sigma`, and return x0 predictoin + + The returned function is suitable for use in scenarios where a denoised state is required based on both conditioned and unconditioned inputs, with an adjustable level of guidance influence. + """ + # data_batch should be the one processed by self.get_data_and_condition + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + # Add conditions for long video generation. + + if condition_latent is None: + condition_latent = torch.zeros(data_batch["latent_hint"].shape, **self.tensor_kwargs) + num_condition_t = 0 + condition_video_augment_sigma_in_inference = 1000 + + condition.video_cond_bool = True + condition = self.add_condition_video_indicator_and_video_input_mask( + condition_latent, condition, num_condition_t + ) + + uncondition.video_cond_bool = True # Not do cfg on condition frames + uncondition = self.add_condition_video_indicator_and_video_input_mask( + condition_latent, uncondition, num_condition_t + ) + + # Add extra conditions for ctrlnet. + latent_hint = data_batch["latent_hint"] + hint_key = data_batch["hint_key"] + setattr(condition, hint_key, latent_hint) + setattr(uncondition, "hint_key", hint_key) + setattr(condition, "hint_key", hint_key) + setattr(condition, "base_model", self.base_model) + setattr(uncondition, "base_model", self.base_model) + + if "use_none_hint" in data_batch and data_batch["use_none_hint"]: + setattr(uncondition, hint_key, None) + else: + setattr(uncondition, hint_key, latent_hint) + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + condition.base_model = self.base_model + uncondition.base_model = self.base_model + cond_x0, eps_pred_cond, logvar_cond = self.denoise( + noise_x, + sigma, + condition, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + ) + uncond_x0, eps_pred_uncond, logvar_uncond = self.denoise( + noise_x, + sigma, + uncondition, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + ) + return cond_x0 + guidance * (cond_x0 - uncond_x0) + + return x0_fn + + def get_x_from_clean( + self, + in_clean_img: torch.Tensor, + sigma_max: float | None, + seed: int = 1, + ) -> Tensor: + """ + in_clean_img (torch.Tensor): input clean image for image-to-image/video-to-video by adding noise then denoising + sigma_max (float): maximum sigma applied to in_clean_image for image-to-image/video-to-video + """ + if in_clean_img is None: + return None + generator = torch.Generator(device=self.tensor_kwargs["device"]) + generator.manual_seed(seed) + noise = torch.randn(*in_clean_img.shape, **self.tensor_kwargs, generator=generator) + if sigma_max is None: + sigma_max = self.sde.sigma_max + x_sigma_max = in_clean_img + noise * sigma_max + return x_sigma_max + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + condition_latent: Union[torch.tensor, None] = None, + num_condition_t: Union[int, None] = None, + condition_video_augment_sigma_in_inference: float = None, + add_input_frames_guidance: bool = False, + solver_option: COMMON_SOLVER_OPTIONS = "2ab", + ) -> Tensor: + """ + Generate samples from the batch. Based on given batch, it will automatically determine whether to generate image or video samples. + """ + + is_image_batch = self.is_image_batch(data_batch) + if n_sample is None: + input_key = self.input_image_key if is_image_batch else self.input_data_key + n_sample = data_batch[input_key].shape[0] + + if self._noise_generator is None: + self._initialize_generators() + + state_shape = list(state_shape) + np.random.seed(self.seed) + x_sigma_max = self.get_x_from_clean((condition_latent), self.sde.sigma_max) + + cp_enabled = parallel_state.get_context_parallel_world_size() > 1 + condition_latent = torch.zeros_like(x_sigma_max) + data_batch["condition_latent"] = condition_latent + x0_fn = self.get_x0_fn_from_batch_with_condition_latent( + data_batch, + guidance, + is_negative_prompt=is_negative_prompt, + condition_latent=condition_latent, + num_condition_t=num_condition_t, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + # add_input_frames_guidance=add_input_frames_guidance, + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + x_sigma_max = split_inputs_cp(x=x_sigma_max, seq_dim=2, cp_group=cp_group) + + samples = None + if self.sampler_type == "EDM": + samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=self.sde.sigma_max) + elif self.sampler_type == "RES": + samples = self.sampler( + x0_fn, x_sigma_max, sigma_max=self.sde.sigma_max, num_steps=num_steps, solver_option=solver_option + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + samples = cat_outputs_cp(samples, seq_dim=2, cp_group=cp_group) + + return samples diff --git a/nemo_vfm/diffusion/sampler/cosmos/cosmos_diffusion_pipeline.py b/nemo_vfm/diffusion/sampler/cosmos/cosmos_diffusion_pipeline.py new file mode 100644 index 00000000..044734ef --- /dev/null +++ b/nemo_vfm/diffusion/sampler/cosmos/cosmos_diffusion_pipeline.py @@ -0,0 +1,504 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0116,C0301 + +import warnings +from typing import Callable, Dict, Tuple + +import numpy as np +import torch +import torch.distributed +from einops import rearrange +from megatron.core import parallel_state +from nemo.collections.diffusion.sampler.batch_ops import * +from nemo.collections.diffusion.sampler.conditioner import BaseVideoCondition, DataType, Edify4Condition +from nemo.collections.diffusion.sampler.context_parallel import cat_outputs_cp, split_inputs_cp +from nemo.collections.diffusion.sampler.edm.edm import EDMSDE, EDMSampler, EDMScaling +from nemo.collections.diffusion.sampler.edm.edm_pipeline import EDMPipeline +from nemo.collections.diffusion.sampler.res.res_sampler import COMMON_SOLVER_OPTIONS, RESSampler +from torch import Tensor + + +# key to check if the video data is normalized or image data is converted to video data +# to avoid apply normalization or augment image dimension multiple times +# It is due to we do not have normalization and augment image dimension in the dataloader and move it to the model +IS_PREPROCESSED_KEY = "is_preprocessed" + + +class CosmosDiffusionPipeline(EDMPipeline): + """ + Diffusion pipeline for Cosmos Diffusion model. + """ + + def __init__( + self, + # Video Tokenizer + # DiT Model + net=None, + # Conditioning Embedders + conditioner=None, + vae=None, + # SDE Args + p_mean=0.0, + p_std=1.0, + sigma_max=80, + sigma_min=0.0002, + sampler_type="RES", # or "EDM" + # EDM Scaling Args + sigma_data=0.5, + seed=1, + loss_add_logvar=True, + ): + super().__init__( + net, + ) + self.vae = vae + self.net = net + self.conditioner = conditioner + + self.p_mean = p_mean + self.p_std = p_std + self.sigma_max = sigma_max + self.sigma_min = sigma_min + self.sigma_data = sigma_data + self.sampler_type = sampler_type + + self.seed = seed + self._noise_generator = None + self._noise_level_generator = None + + self.sde = EDMSDE(p_mean, p_std, sigma_max, sigma_min) + + if self.sampler_type == "EDM": + self.sampler = EDMSampler() + elif self.sampler_type == "RES": + self.sampler = RESSampler() + + self.scaling = EDMScaling(sigma_data) + + self.input_data_key = "video" + self.input_image_key = "images_1024" + self.tensor_kwargs = {"device": "cuda", "dtype": torch.bfloat16} + self.loss_reduce = "mean" + + self.loss_add_logvar = loss_add_logvar + self.loss_scale = 1.0 + + self.aesthetic_finetuning = None + self.camera_sample_weight = None + self.loss_mask_enabled = False + + @property + def noise_level_generator(self): + return self._noise_level_generator + + def _initialize_generators(self): + noise_seed = self.seed + 100 * parallel_state.get_data_parallel_rank(with_context_parallel=True) + noise_level_seed = self.seed + 100 * parallel_state.get_data_parallel_rank(with_context_parallel=False) + self._noise_generator = torch.Generator(device="cuda") + self._noise_generator.manual_seed(noise_seed) + self._noise_level_generator = np.random.default_rng(noise_level_seed) + self.sde._generator = self._noise_level_generator + + def is_image_batch(self, data_batch: dict[str, Tensor]) -> bool: + """We hanlde two types of data_batch. One comes from a joint_dataloader where "dataset_name" can be used to differenciate image_batch and video_batch. + Another comes from a dataloader which we by default assumes as video_data for video model training. + """ + is_image = self.input_image_key in data_batch + is_video = self.input_data_key in data_batch + assert is_image != is_video, ( + "Only one of the input_image_key or input_data_key should be present in the data_batch." + ) + return is_image + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + return self.vae.encode(state) * self.sigma_data + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + return self.vae.decode(latent / self.sigma_data) + + def training_step( + self, data_batch: dict[str, torch.Tensor], iteration: int + ) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + """ + Performs a single training step for the diffusion model. + + This method is responsible for executing one iteration of the model's training. It involves: + 1. Adding noise to the input data using the SDE process. + 2. Passing the noisy data through the network to generate predictions. + 3. Computing the loss based on the difference between the predictions and the original data, \ + considering any configured loss weighting. + + Args: + data_batch (dict): raw data batch draw from the training data loader. + iteration (int): Current iteration number. + + Returns: + tuple: A tuple containing two elements: + - dict: additional data that used to debug / logging / callbacks + - Tensor: The computed loss for the training step as a PyTorch Tensor. + + Raises: + AssertionError: If the class is conditional, \ + but no number of classes is specified in the network configuration. + + Notes: + - The method handles different types of conditioning + - The method also supports Kendall's loss + """ + # Get the input data to noise and denoise~(image, video) and the corresponding conditioner. + x0_from_data_batch, x0, condition = self.get_data_and_condition(data_batch) + + # Sample pertubation noise levels and N(0, 1) noises + sigma, epsilon = self.draw_training_sigma_and_epsilon(x0.size(), condition) + + if parallel_state.is_pipeline_last_stage(): + output_batch, kendall_loss, pred_mse, edm_loss = self.compute_loss_with_epsilon_and_sigma( + data_batch, x0_from_data_batch, x0, condition, epsilon, sigma + ) + return output_batch, kendall_loss + else: + net_output = self.compute_loss_with_epsilon_and_sigma( + data_batch, x0_from_data_batch, x0, condition, epsilon, sigma + ) + return net_output + + def denoise(self, xt: torch.Tensor, sigma: torch.Tensor, condition: Edify4Condition): + """ + Performs denoising on the input noise data, noise level, and condition + + Args: + xt (torch.Tensor): The input noise data. + sigma (torch.Tensor): The noise level. + condition (Edify4Condition): conditional information, generated from self.conditioner + + Returns: + DenoisePrediction: The denoised prediction, it includes clean data predicton (x0), \ + noise prediction (eps_pred) and optional confidence (logvar). + """ + + # Currently only supports video. + # if self.config.get("use_dummy_temporal_dim", False): + # # When using video DiT model for image, we need to use a dummy temporal dimension. + # xt = xt.unsqueeze(2) + + xt = xt.to(**self.tensor_kwargs) + sigma = sigma.to(**self.tensor_kwargs) + # get precondition for the network + c_skip, c_out, c_in, c_noise = self.scaling(sigma=sigma) + + output = self.net( + x=batch_mul(c_in, xt), # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + timesteps=c_noise, # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + **condition.to_dict(), + ) + if isinstance(output, tuple): + net_output, logvar = output + else: + net_output, logvar = output, None + if not parallel_state.is_pipeline_last_stage(): + return net_output + + # logvar = self.net.compute_logvar(c_noise) + + x0_pred = batch_mul(c_skip, xt) + batch_mul(c_out, net_output) + + # get noise prediction based on sde + eps_pred = batch_mul(xt - x0_pred, 1.0 / sigma) + + # Currently only supports video. + # if self.config.get("use_dummy_temporal_dim", False): + # x0_pred = x0_pred.squeeze(2) + # eps_pred = eps_pred.squeeze(2) + + return x0_pred, eps_pred, logvar + + def compute_loss_with_epsilon_and_sigma( + self, + data_batch: dict[str, torch.Tensor], + x0_from_data_batch: torch.Tensor, + x0: torch.Tensor, + condition: Edify4Condition, + epsilon: torch.Tensor, + sigma: torch.Tensor, + ): + """ + Compute loss givee epsilon and sigma + + This method is responsible for computing loss give epsilon and sigma. It involves: + 1. Adding noise to the input data using the SDE process. + 2. Passing the noisy data through the network to generate predictions. + 3. Computing the loss based on the difference between the predictions and the original data, \ + considering any configured loss weighting. + + Args: + data_batch (dict): raw data batch draw from the training data loader. + x0_from_data_batch: raw image/video + x0: image/video latent + condition: text condition + epsilon: noise + sigma: noise level + + Returns: + tuple: A tuple containing four elements: + - dict: additional data that used to debug / logging / callbacks + - Tensor 1: kendall loss, + - Tensor 2: MSE loss, + - Tensor 3: EDM loss + + Raises: + AssertionError: If the class is conditional, \ + but no number of classes is specified in the network configuration. + + Notes: + - The method handles different types of conditioning + - The method also supports Kendall's loss + """ + # Get the mean and stand deviation of the marginal probability distribution. + mean, std = self.sde.marginal_prob(x0, sigma) + # Generate noisy observations + xt = mean + batch_mul(std, epsilon) # corrupted data + + if parallel_state.is_pipeline_last_stage(): + # make prediction + x0_pred, eps_pred, logvar = self.denoise(xt, sigma, condition) + # loss weights for different noise levels + weights_per_sigma = self.get_per_sigma_loss_weights(sigma=sigma) + # extra weight for each sample, for example, aesthetic weight, camera weight + weights_per_sample = self.get_per_sample_weight(data_batch, x0.shape[0]) + loss_mask_per_sample = 1.0 + pred_mse = (x0 - x0_pred) ** 2 * loss_mask_per_sample + edm_loss = batch_mul(pred_mse, weights_per_sigma * weights_per_sample) + if len(edm_loss.shape) > 5: + edm_loss = edm_loss.squeeze(0) + b, c, t, h, w = edm_loss.shape + if logvar is not None and self.loss_add_logvar: + kendall_loss = batch_mul(edm_loss, torch.exp(-logvar).view(-1)).flatten(start_dim=1) + logvar.view( + -1, 1 + ) + else: + kendall_loss = edm_loss.flatten(start_dim=1) + + kendall_loss = rearrange(kendall_loss, "b (c t h w) -> b c (t h w)", b=b, c=c, t=t, h=h, w=w) + + output_batch = { + "x0": x0, + "xt": xt, + "sigma": sigma, + "weights_per_sigma": weights_per_sigma, + "weights_per_sample": weights_per_sample, + "loss_mask_per_sample": loss_mask_per_sample, + "condition": condition, + "model_pred": {"x0_pred": x0_pred, "eps_pred": eps_pred, "logvar": logvar}, + "mse_loss": pred_mse.mean(), + "edm_loss": edm_loss.mean(), + } + return output_batch, kendall_loss, pred_mse, edm_loss + else: + # make prediction + x0_pred = self.denoise(xt, sigma, condition) + return x0_pred.contiguous() + + def get_per_sample_weight(self, data_batch: dict[str, torch.Tensor], batch_size: int): + r""" + extra weight for each sample, for example, aesthetic weight + Args: + data_batch: raw data batch draw from the training data loader. + batch_size: int, the batch size of the input data + """ + aesthetic_cfg = self.aesthetic_finetuning + if (aesthetic_cfg is not None) and getattr(aesthetic_cfg, "enabled", False): + sample_weight = data_batch["aesthetic_weight"] + else: + sample_weight = torch.ones(batch_size, **self.tensor_kwargs) + + camera_cfg = self.camera_sample_weight + if (camera_cfg is not None) and getattr(camera_cfg, "enabled", False): + sample_weight *= 1 + (data_batch["camera_attributes"][:, 1:].sum(dim=1) != 0) * (camera_cfg.weight - 1) + return sample_weight + + def get_per_sigma_loss_weights(self, sigma: torch.Tensor): + """ + Args: + sigma (tensor): noise level + + Returns: + loss weights per sigma noise level + """ + return (sigma**2 + self.sigma_data**2) / (sigma * self.sigma_data) ** 2 + + def get_x0_fn_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + ) -> Callable: + """ + Generates a callable function `x0_fn` based on the provided data batch and guidance factor. + + This function first processes the input data batch through a conditioning workflow (`conditioner`) to obtain conditioned and unconditioned states. It then defines a nested function `x0_fn` which applies a denoising operation on an input `noise_x` at a given noise level `sigma` using both the conditioned and unconditioned states. + + Args: + - data_batch (Dict): A batch of data used for conditioning. The format and content of this dictionary should align with the expectations of the `self.conditioner` + - guidance (float, optional): A scalar value that modulates the influence of the conditioned state relative to the unconditioned state in the output. Defaults to 1.5. + - is_negative_prompt (bool): use negative prompt t5 in uncondition if true + + Returns: + - Callable: A function `x0_fn(noise_x, sigma)` that takes two arguments, `noise_x` and `sigma`, and return x0 predictoin + + The returned function is suitable for use in scenarios where a denoised state is required based on both conditioned and unconditioned inputs, with an adjustable level of guidance influence. + """ + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + cond_x0, _, _ = self.denoise(noise_x, sigma, condition) + uncond_x0, _, _ = self.denoise(noise_x, sigma, uncondition) + return cond_x0 + guidance * (cond_x0 - uncond_x0) + + return x0_fn + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + solver_option: COMMON_SOLVER_OPTIONS = "2ab", + ) -> Tensor: + """ + Generate samples from the batch. Based on given batch, it will automatically determine whether to generate image or video samples. + """ + + is_image_batch = self.is_image_batch(data_batch) + if n_sample is None: + input_key = self.input_image_key if is_image_batch else self.input_data_key + n_sample = data_batch[input_key].shape[0] + if state_shape is None: + if is_image_batch: + state_shape = (self.state_shape[0], 1, *self.state_shape[2:]) # C,T,H,W + + cp_enabled = parallel_state.get_context_parallel_world_size() > 1 + + if self._noise_generator is None: + self._initialize_generators() + + x0_fn = self.get_x0_fn_from_batch(data_batch, guidance, is_negative_prompt=is_negative_prompt) + + state_shape = list(state_shape) + + if len(state_shape) == 4: + state_shape = [1] + state_shape + np.random.seed(self.seed) + x_sigma_max = ( + torch.from_numpy(np.random.randn(*state_shape).astype(np.float32)).to( + dtype=torch.float32, device=self.tensor_kwargs["device"] + ) + * self.sde.sigma_max + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + x_sigma_max = split_inputs_cp(x=x_sigma_max, seq_dim=2, cp_group=cp_group) + + samples = None + if self.sampler_type == "EDM": + samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=self.sde.sigma_max) + elif self.sampler_type == "RES": + samples = self.sampler( + x0_fn, x_sigma_max, sigma_max=self.sde.sigma_max, num_steps=num_steps, solver_option=solver_option + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + samples = cat_outputs_cp(samples, seq_dim=2, cp_group=cp_group) + + return samples + + def _normalize_video_databatch_inplace(self, data_batch: dict[str, Tensor]) -> None: + """ + Normalizes video data in-place on a CUDA device to reduce data loading overhead. + + This function modifies the video data tensor within the provided data_batch dictionary + in-place, scaling the uint8 data from the range [0, 255] to the normalized range [-1, 1]. + + Warning: + A warning is issued if the data has not been previously normalized. + + Args: + data_batch (dict[str, Tensor]): A dictionary containing the video data under a specific key. + This tensor is expected to be on a CUDA device and have dtype of torch.uint8. + + Side Effects: + Modifies the 'input_data_key' tensor within the 'data_batch' dictionary in-place. + + Note: + This operation is performed directly on the CUDA device to avoid the overhead associated + with moving data to/from the GPU. Ensure that the tensor is already on the appropriate device + and has the correct dtype (torch.uint8) to avoid unexpected behaviors. + """ + input_key = self.input_data_key + # only handle video batch + if input_key in data_batch: + # Check if the data has already been normalized and avoid re-normalizing + if IS_PREPROCESSED_KEY in data_batch and data_batch[IS_PREPROCESSED_KEY] is True: + return + else: + data_batch[input_key] = data_batch[input_key].to(dtype=torch.uint8) + assert data_batch[input_key].dtype == torch.uint8, "Video data is not in uint8 format." + warnings.warn("Normalizing video data in-place.") + data_batch[input_key] = data_batch[input_key].to(**self.tensor_kwargs) / 127.5 - 1.0 + data_batch[IS_PREPROCESSED_KEY] = True + + def _augment_image_dim_inplace(self, data_batch: dict[str, Tensor]) -> None: + input_key = self.input_image_key + if input_key in data_batch: + # Check if the data has already been augmented and avoid re-augmenting + if IS_PREPROCESSED_KEY in data_batch and data_batch[IS_PREPROCESSED_KEY] is True: + assert data_batch[input_key].shape[2] == 1, ( + f"Image data is claimed be augmented while its shape is {data_batch[input_key].shape}" + ) + return + else: + data_batch[input_key] = rearrange(data_batch[input_key], "b c h w -> b c 1 h w").contiguous() + data_batch[IS_PREPROCESSED_KEY] = True + + def get_data_and_condition(self, data_batch: dict[str, Tensor]) -> Tuple[Tensor, BaseVideoCondition]: + """ + Retrieves data and conditioning for model input. + + Args: + data_batch: Batch of input data. + + Returns: + Raw data, latent data, and conditioning information. + """ + # Extract video tensor + raw_state = data_batch["video"] * self.sigma_data + # Assume data is already encoded + latent_state = raw_state + + # Condition + condition = self.conditioner(data_batch) + condition.data_type = DataType.VIDEO + + return raw_state, latent_state, condition diff --git a/nemo_vfm/diffusion/sampler/cosmos/cosmos_extended_diffusion_pipeline.py b/nemo_vfm/diffusion/sampler/cosmos/cosmos_extended_diffusion_pipeline.py new file mode 100755 index 00000000..710fb253 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/cosmos/cosmos_extended_diffusion_pipeline.py @@ -0,0 +1,772 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0116,C0301 + + +import warnings +from statistics import NormalDist +from typing import Any, Callable, Dict, Tuple, Union + +import numpy as np +import torch +import torch.distributed +from einops import rearrange +from megatron.core import parallel_state +from nemo.collections.diffusion.sampler.batch_ops import * +from nemo.collections.diffusion.sampler.conditioner import ( + BaseVideoCondition, + DataType, + Edify4Condition, + VideoExtendCondition, +) +from nemo.collections.diffusion.sampler.context_parallel import cat_outputs_cp, split_inputs_cp +from nemo.collections.diffusion.sampler.edm.edm import EDMSDE, EDMSampler, EDMScaling +from nemo.collections.diffusion.sampler.res.res_sampler import COMMON_SOLVER_OPTIONS, RESSampler +from torch import Tensor + + +# key to check if the video data is normalized or image data is converted to video data +# to avoid apply normalization or augment image dimension multiple times +# It is due to we do not have normalization and augment image dimension in the dataloader and move it to the model +IS_PREPROCESSED_KEY = "is_preprocessed" + + +class ExtendedDiffusionPipeline: + """ + Diffusion pipeline for EDM sampling. + Currently only supports video diffusion inference. + """ + + def __init__( + self, + # Video Tokenizer + # DiT Model + net=None, + # Conditioning Embedders + conditioner=None, + vae=None, + # SDE Args + p_mean=0.0, + p_std=1.0, + sigma_max=80, + sigma_min=0.0002, + sampler_type="RES", # or "RES" + # EDM Scaling Args + sigma_data=0.5, + seed=1234, + loss_add_logvar=True, + ): + self.vae = vae + self.net = net + self.conditioner = conditioner + + self.p_mean = p_mean + self.p_std = p_std + self.sigma_max = sigma_max + self.sigma_min = sigma_min + self.sigma_data = sigma_data + self.sampler_type = sampler_type + + self.seed = seed + self._noise_generator = None + self._noise_level_generator = None + + self.sde = EDMSDE(p_mean, p_std, sigma_max, sigma_min) + + if self.sampler_type == "EDM": + self.sampler = EDMSampler() + elif self.sampler_type == "RES": + self.sampler = RESSampler() + + self.scaling = EDMScaling(sigma_data) + + self.input_data_key = "video" + self.input_image_key = "images_1024" + self.tensor_kwargs = {"device": "cuda", "dtype": torch.bfloat16} + self.loss_reduce = "mean" + + self.loss_add_logvar = loss_add_logvar + self.loss_scale = 1.0 + + self.aesthetic_finetuning = None + self.camera_sample_weight = None + self.loss_mask_enabled = False + + @property + def noise_level_generator(self): + return self._noise_level_generator + + def _initialize_generators(self): + noise_seed = self.seed + noise_level_seed = self.seed + self._noise_generator = torch.Generator(device="cuda") + self._noise_generator.manual_seed(noise_seed) + self._noise_level_generator = np.random.default_rng(noise_level_seed) + self.sde._generator = self._noise_level_generator + + def is_image_batch(self, data_batch: dict[str, Tensor]) -> bool: + """We hanlde two types of data_batch. One comes from a joint_dataloader where "dataset_name" can be used to differenciate image_batch and video_batch. + Another comes from a dataloader which we by default assumes as video_data for video model training. + """ + is_image = self.input_image_key in data_batch + is_video = self.input_data_key in data_batch + assert is_image != is_video, ( + "Only one of the input_image_key or input_data_key should be present in the data_batch." + ) + return is_image + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + return self.vae.encode(state) * self.sigma_data + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + return self.vae.decode(latent / self.sigma_data) + + def training_step( + self, data_batch: dict[str, torch.Tensor], iteration: int + ) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + """ + Performs a single training step for the diffusion model. + + This method is responsible for executing one iteration of the model's training. It involves: + 1. Adding noise to the input data using the SDE process. + 2. Passing the noisy data through the network to generate predictions. + 3. Computing the loss based on the difference between the predictions and the original data, \ + considering any configured loss weighting. + + Args: + data_batch (dict): raw data batch draw from the training data loader. + iteration (int): Current iteration number. + + Returns: + tuple: A tuple containing two elements: + - dict: additional data that used to debug / logging / callbacks + - Tensor: The computed loss for the training step as a PyTorch Tensor. + + Raises: + AssertionError: If the class is conditional, \ + but no number of classes is specified in the network configuration. + + Notes: + - The method handles different types of conditioning + - The method also supports Kendall's loss + """ + # Get the input data to noise and denoise~(image, video) and the corresponding conditioner. + x0_from_data_batch, x0, condition = self.get_data_and_condition(data_batch) + # Sample pertubation noise levels and N(0, 1) noises + sigma, epsilon = self.draw_training_sigma_and_epsilon(x0.size(), condition) + output_batch, kendall_loss, pred_mse, edm_loss = self.compute_loss_with_epsilon_and_sigma( + data_batch, x0_from_data_batch, x0, condition, epsilon, sigma + ) + return output_batch, kendall_loss + + def denoise( + self, + xt: torch.Tensor, + sigma: torch.Tensor, + condition: VideoExtendCondition, + condition_video_augment_sigma_in_inference: float = 0.001, + ): + """ + Performs denoising on the input noise data, noise level, and condition + + Args: + xt (torch.Tensor): The input noise data. + sigma (torch.Tensor): The noise level. + condition (Edify4Condition): conditional information, generated from self.conditioner + + Returns: + DenoisePrediction: The denoised prediction, it includes clean data predicton (x0), \ + noise prediction (eps_pred) and optional confidence (logvar). + """ + + xt = xt.to(**self.tensor_kwargs) + sigma = sigma.to(**self.tensor_kwargs) + + gt_latent = condition.gt_latent + condition_latent = gt_latent + condition, augment_latent = self.augment_conditional_latent_frames( + condition, condition_latent, condition_video_augment_sigma_in_inference, sigma + ) + + # Use xt and condition to get new_noise_xt + condition_video_indicator = condition.condition_video_indicator # [B, 1, T, 1, 1] + if parallel_state.get_context_parallel_world_size() > 1: + cp_group = parallel_state.get_context_parallel_group() + condition_video_indicator = split_inputs_cp(condition_video_indicator, seq_dim=2, cp_group=cp_group) + augment_latent = split_inputs_cp(augment_latent, seq_dim=2, cp_group=cp_group) + gt_latent = split_inputs_cp(gt_latent, seq_dim=2, cp_group=cp_group) + condition.condition_video_input_mask = split_inputs_cp( + condition.condition_video_input_mask, seq_dim=2, cp_group=cp_group + ) + if condition.condition_video_pose is not None: + condition.condition_video_pose = split_inputs_cp( + condition.condition_video_pose, seq_dim=2, cp_group=cp_group + ) + + # Combine / concatenate the conditional video latent with the noisy input. + new_noise_xt = condition_video_indicator * augment_latent + (1 - condition_video_indicator) * xt + new_noise_xt = new_noise_xt.to(**self.tensor_kwargs) + + # get precondition for the network + c_skip, c_out, c_in, c_noise = self.scaling(sigma=sigma) + + condition.data_type = DataType.VIDEO + + output = self.net( + x=batch_mul(c_in, new_noise_xt), # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + timesteps=c_noise, # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + **condition.to_dict(), + ) + if isinstance(output, tuple): + net_output, logvar = output + else: + net_output, logvar = output, None + + if not parallel_state.is_pipeline_last_stage(): + return net_output + + x0_pred = batch_mul(c_skip, new_noise_xt) + batch_mul(c_out, net_output) + + # get noise prediction based on sde + eps_pred = batch_mul(new_noise_xt - x0_pred, 1.0 / sigma) + + x0_pred_replaced = condition_video_indicator * gt_latent + (1 - condition_video_indicator) * x0_pred + x0_pred = x0_pred_replaced + + return x0_pred, eps_pred, logvar + + def compute_loss_with_epsilon_and_sigma( + self, + data_batch: dict[str, torch.Tensor], + x0_from_data_batch: torch.Tensor, + x0: torch.Tensor, + condition: Edify4Condition, + epsilon: torch.Tensor, + sigma: torch.Tensor, + ): + """ + Compute loss given epsilon and sigma + + This method is responsible for computing loss give epsilon and sigma. It involves: + 1. Adding noise to the input data using the SDE process. + 2. Passing the noisy data through the network to generate predictions. + 3. Computing the loss based on the difference between the predictions and the original data, \ + considering any configured loss weighting. + + Args: + data_batch (dict): raw data batch draw from the training data loader. + x0_from_data_batch: raw image/video + x0: image/video latent + condition: text condition + epsilon: noise + sigma: noise level + + Returns: + tuple: A tuple containing four elements: + - dict: additional data that used to debug / logging / callbacks + - Tensor 1: kendall loss, + - Tensor 2: MSE loss, + - Tensor 3: EDM loss + + Raises: + AssertionError: If the class is conditional, \ + but no number of classes is specified in the network configuration. + + Notes: + - The method handles different types of conditioning + - The method also supports Kendall's loss + """ + # Get the mean and stand deviation of the marginal probability distribution. + mean, std = self.sde.marginal_prob(x0, sigma) + # Generate noisy observations + xt = mean + batch_mul(std, epsilon) # corrupted data + + # make prediction + x0_pred, eps_pred, logvar = self.denoise(xt, sigma, condition) + # loss weights for different noise levels + weights_per_sigma = self.get_per_sigma_loss_weights(sigma=sigma) + # extra weight for each sample, for example, aesthetic weight, camera weight + weights_per_sample = self.get_per_sample_weight(data_batch, x0.shape[0]) + # extra loss mask for each sample, for example, human faces, hands + # loss_mask_per_sample = self.get_per_sample_loss_mask(data_batch, x0_from_data_batch.shape, x0.shape) + loss_mask_per_sample = 1.0 + + # Compute loss kernel. + pred_mse = (x0 - x0_pred) ** 2 * loss_mask_per_sample # Equation 5. + + edm_loss = batch_mul(pred_mse, weights_per_sigma * weights_per_sample) + if len(edm_loss.shape) > 5: + edm_loss = edm_loss.squeeze(0) + b, c, t, h, w = edm_loss.shape + if logvar is not None and self.loss_add_logvar: + kendall_loss = batch_mul(edm_loss, torch.exp(-logvar).view(-1)).flatten(start_dim=1) + logvar.view(-1, 1) + else: + kendall_loss = edm_loss.flatten(start_dim=1) + + kendall_loss = rearrange(kendall_loss, "b (c t h w) -> b c (t h w)", b=b, c=c, t=t, h=h, w=w) + + output_batch = { + "x0": x0, + "xt": xt, + "sigma": sigma, + "weights_per_sigma": weights_per_sigma, + "weights_per_sample": weights_per_sample, + "loss_mask_per_sample": loss_mask_per_sample, + "condition": condition, + "model_pred": {"x0_pred": x0_pred, "eps_pred": eps_pred, "logvar": logvar}, + "mse_loss": pred_mse.mean(), + "edm_loss": edm_loss.mean(), + } + + return output_batch, kendall_loss, pred_mse, edm_loss + + def get_per_sample_weight(self, data_batch: dict[str, torch.Tensor], batch_size: int): + r""" + extra weight for each sample, for example, aesthetic weight + Args: + data_batch: raw data batch draw from the training data loader. + batch_size: int, the batch size of the input data + """ + return 1.0 + + def get_per_sample_loss_mask(self, data_batch, raw_x_shape, latent_x_shape): + """ + extra loss mask for each sample, for example, human faces, hands. + + Args: + data_batch (dict): raw data batch draw from the training data loader. + raw_x_shape (tuple): shape of the input data. We need the raw_x_shape for necessary resize operation. + latent_x_shape (tuple): shape of the latent data + """ + # if self.loss_mask_enabled: + # raw_x_shape = [raw_x_shape[0], 1, *raw_x_shape[2:]] + # weights = create_per_sample_loss_mask( + # self.loss_masking, data_batch, raw_x_shape, torch.get_default_dtype(), "cuda" + # ) + # return F.interpolate(weights, size=latent_x_shape[2:], mode="bilinear") + + return 1.0 + + def get_per_sigma_loss_weights(self, sigma: torch.Tensor): + """ + Args: + sigma (tensor): noise level + + Returns: + loss weights per sigma noise level + """ + return (sigma**2 + self.sigma_data**2) / (sigma * self.sigma_data) ** 2 + + def get_x0_fn_from_batch_with_condition_latent( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + condition_latent: torch.Tensor = None, + num_condition_t: Union[int, None] = None, + condition_video_augment_sigma_in_inference: float = None, + add_input_frames_guidance: bool = False, + ) -> Callable: + """Creates a denoising function that incorporates latent conditioning for video generation. + + This function extends the base model's denoising by adding latent conditioning. It processes + the input batch to create conditioned and unconditioned states, then returns a function that + performs denoising with classifier-free guidance. + + Args: + data_batch: Input data dictionary for conditioning + guidance: Classifier-free guidance scale (default: 1.5) + is_negative_prompt: Whether to use negative T5 prompt for unconditioned generation + condition_latent: Video latent tensor of shape (B,C,T,H,W) used as condition + num_condition_t: Number of timesteps to condition on when using "first_n" conditioning + condition_video_augment_sigma_in_inference: Noise level for augmenting condition video + add_input_frames_guidance: Whether to apply classifier-free guidance to input frames + + Returns: + A function that takes noisy input x and noise level sigma and returns denoised prediction + """ + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + + condition.video_cond_bool = True + condition = self.add_condition_video_indicator_and_video_input_mask( + condition_latent, condition, num_condition_t + ) + + if "plucker_embeddings" in data_batch: + condition.add_pose_condition = True + condition = self.add_condition_pose(data_batch, condition) + + uncondition.video_cond_bool = False if add_input_frames_guidance else True + uncondition = self.add_condition_video_indicator_and_video_input_mask( + condition_latent, uncondition, num_condition_t + ) + + if condition.add_pose_condition: + uncondition = self.add_condition_pose(data_batch, uncondition) + + condition_video_input_mask_copy = condition.condition_video_input_mask.clone().detach() + uncondition_video_input_mask_copy = uncondition.condition_video_input_mask.clone().detach() + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + condition.condition_video_input_mask = condition_video_input_mask_copy.clone().detach() + cond_x0_pred, _, _ = self.denoise( + noise_x, + sigma, + condition, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + ) + uncondition.condition_video_input_mask = uncondition_video_input_mask_copy.clone().detach() + uncond_x0_pred, _, _ = self.denoise( + noise_x, + sigma, + uncondition, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + ) + + return cond_x0_pred + guidance * (cond_x0_pred - uncond_x0_pred) + + return x0_fn + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + condition_latent: Union[torch.tensor, None] = None, + num_condition_t: Union[int, None] = None, + condition_video_augment_sigma_in_inference: float = None, + add_input_frames_guidance: bool = False, + solver_option: COMMON_SOLVER_OPTIONS = "2ab", + ) -> Tensor: + """ + Generate samples from the batch. Based on given batch, it will automatically determine whether to generate image or video samples. + """ + + is_image_batch = self.is_image_batch(data_batch) + if n_sample is None: + input_key = self.input_image_key if is_image_batch else self.input_data_key + n_sample = data_batch[input_key].shape[0] + + if self._noise_generator is None: + self._initialize_generators() + + x0_fn = self.get_x0_fn_from_batch_with_condition_latent( + data_batch, + guidance, + is_negative_prompt=is_negative_prompt, + condition_latent=condition_latent, + num_condition_t=num_condition_t, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + add_input_frames_guidance=add_input_frames_guidance, + ) + + state_shape = list(state_shape) + np.random.seed(self.seed) + x_sigma_max = ( + torch.from_numpy(np.random.randn(1, *state_shape).astype(np.float32)).to( + dtype=torch.float32, device=self.tensor_kwargs["device"] + ) + * self.sde.sigma_max + ) + + cp_enabled = parallel_state.get_context_parallel_world_size() > 1 + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + x_sigma_max = split_inputs_cp(x=x_sigma_max, seq_dim=2, cp_group=cp_group) + + samples = None + if self.sampler_type == "EDM": + samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=self.sde.sigma_max) + elif self.sampler_type == "RES": + samples = self.sampler( + x0_fn, x_sigma_max, sigma_max=self.sde.sigma_max, num_steps=num_steps, solver_option=solver_option + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + samples = cat_outputs_cp(samples, seq_dim=2, cp_group=cp_group) + + return samples + + def _normalize_video_databatch_inplace(self, data_batch: dict[str, Tensor]) -> None: + """ + Normalizes video data in-place on a CUDA device to reduce data loading overhead. + + This function modifies the video data tensor within the provided data_batch dictionary + in-place, scaling the uint8 data from the range [0, 255] to the normalized range [-1, 1]. + + Warning: + A warning is issued if the data has not been previously normalized. + + Args: + data_batch (dict[str, Tensor]): A dictionary containing the video data under a specific key. + This tensor is expected to be on a CUDA device and have dtype of torch.uint8. + + Side Effects: + Modifies the 'input_data_key' tensor within the 'data_batch' dictionary in-place. + + Note: + This operation is performed directly on the CUDA device to avoid the overhead associated + with moving data to/from the GPU. Ensure that the tensor is already on the appropriate device + and has the correct dtype (torch.uint8) to avoid unexpected behaviors. + """ + input_key = self.input_data_key + # only handle video batch + if input_key in data_batch: + # Check if the data has already been normalized and avoid re-normalizing + if IS_PREPROCESSED_KEY in data_batch and data_batch[IS_PREPROCESSED_KEY] is True: + return + else: + data_batch[input_key] = data_batch[input_key].to(dtype=torch.uint8) + assert data_batch[input_key].dtype == torch.uint8, "Video data is not in uint8 format." + warnings.warn("Normalizing video data in-place.") + data_batch[input_key] = data_batch[input_key].to(**self.tensor_kwargs) / 127.5 - 1.0 + data_batch[IS_PREPROCESSED_KEY] = True + + def _augment_image_dim_inplace(self, data_batch: dict[str, Tensor]) -> None: + input_key = self.input_image_key + if input_key in data_batch: + # Check if the data has already been augmented and avoid re-augmenting + if IS_PREPROCESSED_KEY in data_batch and data_batch[IS_PREPROCESSED_KEY] is True: + assert data_batch[input_key].shape[2] == 1, ( + f"Image data is claimed be augmented while its shape is {data_batch[input_key].shape}" + ) + return + else: + data_batch[input_key] = rearrange(data_batch[input_key], "b c h w -> b c 1 h w").contiguous() + data_batch[IS_PREPROCESSED_KEY] = True + + def draw_training_sigma_and_epsilon(self, x0_size: int, condition: Any) -> torch.Tensor: + del condition + batch_size = x0_size[0] + if self._noise_generator is None: + self._initialize_generators() + epsilon = torch.randn(x0_size, **self.tensor_kwargs, generator=self._noise_generator) + self.video_noise_multiplier = 1.0 + sigma_B = self.sde.sample_t(batch_size) * self.video_noise_multiplier + return sigma_B.to(**self.tensor_kwargs), epsilon + + def draw_augment_sigma_and_epsilon( + self, size: int, condition: VideoExtendCondition, p_mean: float, p_std: float, multiplier: float + ) -> Tensor: + del condition + batch_size = size[0] + epsilon = torch.randn(size, **self.tensor_kwargs) + + gaussian_dist = NormalDist(mu=p_mean, sigma=p_std) + cdf_vals = np.random.uniform(size=(batch_size)) + samples_interval_gaussian = [gaussian_dist.inv_cdf(cdf_val) for cdf_val in cdf_vals] + + log_sigma = torch.tensor(samples_interval_gaussian, device="cuda") + sigma_B = torch.exp(log_sigma) * multiplier + + return sigma_B.to(**self.tensor_kwargs), epsilon + + def get_data_and_condition( + self, + data_batch: dict[str, Tensor], + num_condition_t: Union[int, None] = None, + ) -> Tuple[Tensor, BaseVideoCondition]: + # Latent state + raw_state = data_batch["video"] * self.sigma_data + # assume data is already encoded + latent_state = raw_state + # latent_state = self.encode(raw_state) + + # Condition + condition = self.conditioner(data_batch) + condition.data_type = DataType.VIDEO + + condition = self.add_condition_video_indicator_and_video_input_mask( + latent_state, condition, num_condition_t=data_batch["num_condition_t"] + ) + + if "plucker_embeddings" in data_batch: + condition.add_pose_condition = True + condition.first_random_n_num_condition_t_max = 1 + condition = self.add_condition_pose(data_batch, condition) + + return raw_state, latent_state, condition + + def add_condition_video_indicator_and_video_input_mask( + self, latent_state: torch.Tensor, condition: VideoExtendCondition, num_condition_t: Union[int, None] = None + ) -> VideoExtendCondition: + """Add condition_video_indicator and condition_video_input_mask to the condition object for video conditioning. + condition_video_indicator is a binary tensor indicating the condition region in the latent state. 1x1xTx1x1 tensor. + condition_video_input_mask will be concat with the input for the network. + Args: + latent_state (torch.Tensor): latent state tensor in shape B,C,T,H,W + condition (VideoExtendCondition): condition object + num_condition_t (int): number of condition latent T, used in inference to decide the condition region and config.conditioner.video_cond_bool.condition_location == "first_n" + Returns: + VideoExtendCondition: updated condition object + """ + T = latent_state.shape[2] + latent_dtype = latent_state.dtype + condition_video_indicator = torch.zeros(1, 1, T, 1, 1, device=latent_state.device).type( + latent_dtype + ) # 1 for condition region + if condition.condition_location == "first_n": + # Only in inference to decide the condition region + assert num_condition_t is not None, "num_condition_t should be provided" + assert num_condition_t <= T, f"num_condition_t should be less than T, get {num_condition_t}, {T}" + condition_video_indicator[:, :, :num_condition_t] += 1.0 + elif condition.condition_location == "first_random_n": + # Only in training + num_condition_t_max = condition.first_random_n_num_condition_t_max + assert num_condition_t_max <= T, ( + f"num_condition_t_max should be less than T, get {num_condition_t_max}, {T}" + ) + assert num_condition_t_max >= condition.first_random_n_num_condition_t_min + num_condition_t = torch.randint( + condition.first_random_n_num_condition_t_min, + num_condition_t_max + 1, + (1,), + ).item() + condition_video_indicator[:, :, :num_condition_t] += 1.0 + + elif condition.condition_location == "random": + # Only in training + condition_rate = condition.random_conditon_rate + flag = torch.ones(1, 1, T, 1, 1, device=latent_state.device).type(latent_dtype) * condition_rate + condition_video_indicator = torch.bernoulli(flag).type(latent_dtype).to(latent_state.device) + else: + raise NotImplementedError( + f"condition_location {condition.condition_location} not implemented; training={self.training}" + ) + condition.gt_latent = latent_state + condition.condition_video_indicator = condition_video_indicator + + B, C, T, H, W = latent_state.shape + # Create additional input_mask channel, this will be concatenated to the input of the network + # See design doc section (Implementation detail A.1 and A.2) for visualization + ones_padding = torch.ones((B, 1, T, H, W), dtype=latent_state.dtype, device=latent_state.device) + zeros_padding = torch.zeros((B, 1, T, H, W), dtype=latent_state.dtype, device=latent_state.device) + assert condition.video_cond_bool is not None, "video_cond_bool should be set" + + # The input mask indicate whether the input is conditional region or not + if condition.video_cond_bool: # Condition one given video frames + condition.condition_video_input_mask = ( + condition_video_indicator * ones_padding + (1 - condition_video_indicator) * zeros_padding + ) + else: # Unconditional case, use for cfg + condition.condition_video_input_mask = zeros_padding + + return condition + + def augment_conditional_latent_frames( + self, + condition: VideoExtendCondition, + gt_latent: Tensor, + condition_video_augment_sigma_in_inference: float = 0.001, + sigma: Tensor = None, + ) -> Union[VideoExtendCondition, Tensor]: + """This function is used to augment the condition input with noise + Args: + condition (VideoExtendCondition): condition object + condition_video_indicator: binary tensor indicating the region is condition(value=1) or generation(value=0). Bx1xTx1x1 tensor. + condition_video_input_mask: input mask for the network input, indicating the condition region. B,1,T,H,W tensor. will be concat with the input for the network. + cfg_video_cond_bool (VideoCondBoolConfig): video condition bool config + gt_latent (Tensor): ground truth latent tensor in shape B,C,T,H,W + condition_video_augment_sigma_in_inference (float): sigma for condition video augmentation in inference + sigma (Tensor): noise level for the generation region + Returns: + VideoExtendCondition: updated condition object + condition_video_augment_sigma: sigma for the condition region, feed to the network + augment_latent (Tensor): augmented latent tensor in shape B,C,T,H,W + + """ + + if condition.apply_corruption_to_condition_region == "noise_with_sigma": + # Training only, sample sigma for the condition region + augment_sigma, _ = self.draw_augment_sigma_and_epsilon( + gt_latent.shape, + condition, + condition.augment_sigma_sample_p_mean, + condition.augment_sigma_sample_p_std, + condition.augment_sigma_sample_multiplier, + ) + + elif condition.apply_corruption_to_condition_region == "noise_with_sigma_fixed": + # Inference only, use fixed sigma for the condition region + assert condition_video_augment_sigma_in_inference is not None, ( + "condition_video_augment_sigma_in_inference should be provided" + ) + augment_sigma = condition_video_augment_sigma_in_inference + + if augment_sigma >= sigma.flatten()[0]: + # This is a inference trick! If the sampling sigma is smaller than the augment sigma, we will start denoising the condition region together. + # This is achieved by setting all region as `generation`, i.e. value=0 + condition.condition_video_indicator = condition.condition_video_indicator * 0 + + augment_sigma = torch.tensor([augment_sigma], **self.tensor_kwargs) + + else: + raise ValueError(f"does not support {condition.apply_corruption_to_condition_region}") + + # Now apply the augment_sigma to the gt_latent + noise = torch.randn(*gt_latent.shape, **self.tensor_kwargs) + + augment_latent = gt_latent + noise * augment_sigma[:, None, None, None, None] + + _, _, c_in_augment, c_noise_augment = self.scaling(sigma=augment_sigma) + + if condition.condition_on_augment_sigma: # model takes augment_sigma as input + if condition.condition_video_indicator.sum() > 0: # has condition frames + condition.condition_video_augment_sigma = c_noise_augment + else: # no condition frames + condition.condition_video_augment_sigma = torch.zeros_like(c_noise_augment) + + # Multiply the whole latent with c_in_augment + augment_latent_cin = batch_mul(augment_latent, c_in_augment) + + # Since the whole latent will multiply with c_in later, we divide the value to cancel the effect + _, _, c_in, _ = self.scaling(sigma=sigma) + augment_latent_cin = batch_mul(augment_latent_cin, 1 / c_in) + + return condition, augment_latent_cin + + def add_condition_pose(self, data_batch: Dict, condition: VideoExtendCondition) -> VideoExtendCondition: + """Add pose condition to the condition object. For camera control model + Args: + data_batch (Dict): data batch, with key "plucker_embeddings", in shape B,T,C,H,W + latent_state (torch.Tensor): latent state tensor in shape B,C,T,H,W + condition (VideoExtendCondition): condition object + num_condition_t (int): number of condition latent T, used in inference to decide the condition region and config.conditioner.video_cond_bool.condition_location == "first_n" + Returns: + VideoExtendCondition: updated condition object + """ + assert "plucker_embeddings" in data_batch or "plucker_embeddings_downsample" in data_batch.keys(), ( + f"plucker_embeddings should be in data_batch. only find {data_batch.keys()}" + ) + plucker_embeddings = ( + data_batch["plucker_embeddings"] + if "plucker_embeddings_downsample" not in data_batch.keys() + else data_batch["plucker_embeddings_downsample"] + ) + condition.condition_video_pose = rearrange(plucker_embeddings, "b t c h w -> b c t h w").contiguous() + condition.condition_video_pose = condition.condition_video_pose.to(**self.tensor_kwargs) + + return condition diff --git a/nemo_vfm/diffusion/sampler/edm/__init__.py b/nemo_vfm/diffusion/sampler/edm/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/edm/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/sampler/edm/edm.py b/nemo_vfm/diffusion/sampler/edm/edm.py new file mode 100644 index 00000000..698acbb1 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/edm/edm.py @@ -0,0 +1,137 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from statistics import NormalDist +from typing import Callable, Tuple + +import numpy as np +import torch +from torch import nn +from tqdm import tqdm + + +class EDMScaling: + def __init__(self, sigma_data: float = 0.5): + self.sigma_data = sigma_data + + def __call__(self, sigma: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + c_skip = self.sigma_data**2 / (sigma**2 + self.sigma_data**2) + c_out = sigma * self.sigma_data / (sigma**2 + self.sigma_data**2) ** 0.5 + c_in = 1 / (sigma**2 + self.sigma_data**2) ** 0.5 + c_noise = 0.25 * sigma.log() + return c_skip, c_out, c_in, c_noise + + +class EDMSDE: + def __init__( + self, + p_mean: float = -1.2, + p_std: float = 1.2, + sigma_max: float = 80.0, + sigma_min: float = 0.002, + ): + self.gaussian_dist = NormalDist(mu=p_mean, sigma=p_std) + self.sigma_max = sigma_max + self.sigma_min = sigma_min + self._generator = np.random + + def sample_t(self, batch_size: int) -> torch.Tensor: + cdf_vals = self._generator.uniform(size=(batch_size)) + samples_interval_gaussian = [self.gaussian_dist.inv_cdf(cdf_val) for cdf_val in cdf_vals] + log_sigma = torch.tensor(samples_interval_gaussian, device="cuda") + return torch.exp(log_sigma) + + def marginal_prob(self, x0: torch.Tensor, sigma: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + return x0, sigma + + +class EDMSampler(nn.Module): + """ + Elucidating the Design Space of Diffusion-Based Generative Models (EDM) + # https://github.com/NVlabs/edm/blob/62072d2612c7da05165d6233d13d17d71f213fee/generate.py#L25 + + Attributes: + None + + Methods: + forward(x0_fn: Callable, x_sigma_max: torch.Tensor, num_steps: int = 35, sigma_min: float = 0.002, + sigma_max: float = 80, rho: float = 7, S_churn: float = 0, S_min: float = 0, + S_max: float = float("inf"), S_noise: float = 1) -> torch.Tensor: + Performs the forward pass for the EDM sampling process. + + Parameters: + x0_fn (Callable): A function that takes in a tensor and returns a denoised tensor. + x_sigma_max (torch.Tensor): The initial noise level tensor. + num_steps (int, optional): The number of sampling steps. Default is 35. + sigma_min (float, optional): The minimum noise level. Default is 0.002. + sigma_max (float, optional): The maximum noise level. Default is 80. + rho (float, optional): The rho parameter for time step discretization. Default is 7. + S_churn (float, optional): The churn parameter for noise increase. Default is 0. + S_min (float, optional): The minimum value for the churn parameter. Default is 0. + S_max (float, optional): The maximum value for the churn parameter. Default is float("inf"). + S_noise (float, optional): The noise scale for the churn parameter. Default is 1. + + Returns: + torch.Tensor: The sampled tensor after the EDM process. + """ + + @torch.no_grad() + def forward( + self, + x0_fn: Callable, + x_sigma_max: torch.Tensor, + num_steps: int = 35, + sigma_min: float = 0.002, + sigma_max: float = 80, + rho: float = 7, + S_churn: float = 0, + S_min: float = 0, + S_max: float = float("inf"), + S_noise: float = 1, + ) -> torch.Tensor: + # Time step discretization. + in_dtype = x_sigma_max.dtype + _ones = torch.ones(x_sigma_max.shape[0], dtype=in_dtype, device=x_sigma_max.device) + step_indices = torch.arange(num_steps, dtype=torch.float64, device=x_sigma_max.device) + t_steps = ( + sigma_max ** (1 / rho) + step_indices / (num_steps - 1) * (sigma_min ** (1 / rho) - sigma_max ** (1 / rho)) + ) ** rho + t_steps = torch.cat([t_steps, torch.zeros_like(t_steps[:1])]) # t_N = 0 + + # Main sampling loop. + x_next = x_sigma_max.to(torch.float64) + for i, (t_cur, t_next) in enumerate( + tqdm(zip(t_steps[:-1], t_steps[1:], strict=False), total=len(t_steps) - 1) + ): # 0, ..., N-1 + x_cur = x_next + + # Increase noise temporarily. + gamma = min(S_churn / num_steps, np.sqrt(2) - 1) if S_min <= t_cur <= S_max else 0 + t_hat = t_cur + gamma * t_cur + x_hat = x_cur + (t_hat**2 - t_cur**2).sqrt() * S_noise * torch.randn_like(x_cur) + + # Euler step. + denoised = x0_fn(x_hat.to(in_dtype), t_hat.to(in_dtype) * _ones).to(torch.float64) + d_cur = (x_hat - denoised) / t_hat + x_next = x_hat + (t_next - t_hat) * d_cur + + # Apply 2nd order correction. + if i < num_steps - 1: + denoised = x0_fn(x_hat.to(in_dtype), t_hat.to(in_dtype) * _ones).to(torch.float64) + d_prime = (x_next - denoised) / t_next + x_next = x_hat + (t_next - t_hat) * (0.5 * d_cur + 0.5 * d_prime) + + return x_next.to(in_dtype) diff --git a/nemo_vfm/diffusion/sampler/edm/edm_pipeline.py b/nemo_vfm/diffusion/sampler/edm/edm_pipeline.py new file mode 100644 index 00000000..ecc69c5f --- /dev/null +++ b/nemo_vfm/diffusion/sampler/edm/edm_pipeline.py @@ -0,0 +1,432 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Callable, Dict, Optional, Tuple + +import numpy as np +import torch +import torch.distributed +from megatron.core import parallel_state +from nemo.collections.diffusion.sampler.batch_ops import batch_mul +from nemo.collections.diffusion.sampler.context_parallel import cat_outputs_cp +from nemo.collections.diffusion.sampler.edm.edm import EDMSDE, EDMSampler, EDMScaling +from torch import Tensor + + +class EDMPipeline: + """ + EDMPipeline is a class that implements a diffusion model pipeline for video generation. It includes methods for + initializing the pipeline, encoding and decoding video data, performing training steps, denoising, and generating + samples. + Attributes: + p_mean: Mean for SDE process. + p_std: Standard deviation for SDE process. + sigma_max: Maximum noise level. + sigma_min: Minimum noise level. + _noise_generator: Generator for noise. + _noise_level_generator: Generator for noise levels. + sde: SDE process. + sampler: Sampler for the diffusion model. + scaling: Scaling for EDM. + input_data_key: Key for input video data. + input_image_key: Key for input image data. + tensor_kwargs: Tensor keyword arguments. + loss_reduce: Method for reducing loss. + loss_scale: Scale factor for loss. + aesthetic_finetuning: Aesthetic finetuning parameter. + camera_sample_weight: Camera sample weight parameter. + loss_mask_enabled: Flag for enabling loss mask. + Methods: + noise_level_generator: Returns the noise level generator. + _initialize_generators: Initializes noise and noise-level generators. + encode: Encodes input tensor using the video tokenizer. + decode: Decodes latent tensor using video tokenizer. + training_step: Performs a single training step for the diffusion model. + denoise: Performs denoising on the input noise data, noise level, and condition. + compute_loss_with_epsilon_and_sigma: Computes the loss for training. + get_per_sigma_loss_weights: Returns loss weights per sigma noise level. + get_condition_uncondition: Returns conditioning and unconditioning for classifier-free guidance. + get_x0_fn_from_batch: Creates a function to generate denoised predictions with the sampler. + generate_samples_from_batch: Generates samples based on input data batch. + _normalize_video_databatch_inplace: Normalizes video data in-place on a CUDA device to [-1, 1]. + draw_training_sigma_and_epsilon: Draws training noise (epsilon) and noise levels (sigma). + random_dropout_input: Applies random dropout to the input tensor. + get_data_and_condition: Retrieves data and conditioning for model input. + """ + + def __init__( + self, + net, + vae=None, + p_mean=0.0, + p_std=1.0, + sigma_max=80, + sigma_min=0.0002, + sigma_data=0.5, + seed=1234, + ): + """ + Initializes the EDM pipeline with the given parameters. + + Args: + net: The DiT model. + vae: The Video Tokenizer (optional). + p_mean (float): Mean for the SDE. + p_std (float): Standard deviation for the SDE. + sigma_max (float): Maximum sigma value for the SDE. + sigma_min (float): Minimum sigma value for the SDE. + sigma_data (float): Sigma value for EDM scaling. + seed (int): Random seed for reproducibility. + + Attributes: + vae: The Video Tokenizer. + net: The DiT model. + p_mean (float): Mean for the SDE. + p_std (float): Standard deviation for the SDE. + sigma_max (float): Maximum sigma value for the SDE. + sigma_min (float): Minimum sigma value for the SDE. + sigma_data (float): Sigma value for EDM scaling. + seed (int): Random seed for reproducibility. + _noise_generator: Placeholder for noise generator. + _noise_level_generator: Placeholder for noise level generator. + sde: Instance of EDMSDE initialized with p_mean, p_std, sigma_max, and sigma_min. + sampler: Instance of EDMSampler. + scaling: Instance of EDMScaling initialized with sigma_data. + input_data_key (str): Key for input data. + input_image_key (str): Key for input images. + tensor_kwargs (dict): Tensor keyword arguments for device and dtype. + loss_reduce (str): Method to reduce loss ('mean' or other). + loss_scale (float): Scale factor for loss. + """ + self.vae = vae + self.net = net + + self.p_mean = p_mean + self.p_std = p_std + self.sigma_max = sigma_max + self.sigma_min = sigma_min + self.sigma_data = sigma_data + + self.seed = seed + self._noise_generator = None + self._noise_level_generator = None + + self.sde = EDMSDE(p_mean, p_std, sigma_max, sigma_min) + self.sampler = EDMSampler() + self.scaling = EDMScaling(sigma_data) + + self.input_data_key = "video" + self.input_image_key = "images_1024" + self.tensor_kwargs = {"device": "cuda", "dtype": torch.bfloat16} + self.loss_reduce = "mean" + self.loss_scale = 1.0 + + @property + def noise_level_generator(self): + """ + Generates noise levels for the EDM pipeline. + + Returns: + Callable: A function or generator that produces noise levels. + """ + return self._noise_level_generator + + def _initialize_generators(self): + """ + Initializes the random number generators for noise and noise level. + + This method sets up two generators: + 1. A PyTorch generator for noise, seeded with a combination of the base seed and the data parallel rank. + 2. A NumPy generator for noise levels, seeded similarly but without considering context parallel rank. + + Returns: + None + """ + noise_seed = self.seed + 100 * parallel_state.get_data_parallel_rank(with_context_parallel=True) + noise_level_seed = self.seed + 100 * parallel_state.get_data_parallel_rank(with_context_parallel=False) + self._noise_generator = torch.Generator(device="cuda") + self._noise_generator.manual_seed(noise_seed) + self._noise_level_generator = np.random.default_rng(noise_level_seed) + self.sde._generator = self._noise_level_generator + + def training_step( + self, data_batch: dict[str, torch.Tensor], iteration: int + ) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + """ + Performs a single training step for the diffusion model. + + This method is responsible for executing one iteration of the model's training. It involves: + 1. Adding noise to the input data using the SDE process. + 2. Passing the noisy data through the network to generate predictions. + 3. Computing the loss based on the difference between the predictions and the original data. + + Args: + data_batch (dict): raw data batch draw from the training data loader. + iteration (int): Current iteration number. + + Returns: + A tuple with the output batch and the computed loss. + """ + # Get the input data to noise and denoise~(image, video) and the corresponding conditioner. + x0_from_data_batch, x0, condition = self.get_data_and_condition(data_batch) + + # Sample pertubation noise levels and N(0, 1) noises + sigma, epsilon = self.draw_training_sigma_and_epsilon(x0.size(), condition) + + if parallel_state.is_pipeline_last_stage(): + output_batch, pred_mse, edm_loss = self.compute_loss_with_epsilon_and_sigma( + data_batch, x0_from_data_batch, x0, condition, epsilon, sigma + ) + + return output_batch, edm_loss + else: + net_output = self.compute_loss_with_epsilon_and_sigma( + data_batch, x0_from_data_batch, x0, condition, epsilon, sigma + ) + return net_output + + def denoise(self, xt: torch.Tensor, sigma: torch.Tensor, condition: dict[str, torch.Tensor]): + """ + Performs denoising on the input noise data, noise level, and condition + + Args: + xt (torch.Tensor): The input noise data. + sigma (torch.Tensor): The noise level. + condition (dict[str, torch.Tensor]): conditional information + + Returns: + Predicted clean data (x0) and noise (eps_pred). + """ + + xt = xt.to(**self.tensor_kwargs) + sigma = sigma.to(**self.tensor_kwargs) + # get precondition for the network + c_skip, c_out, c_in, c_noise = self.scaling(sigma=sigma) + + net_output = self.net( + x=batch_mul(c_in, xt), # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + timesteps=c_noise, # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + **condition, + ) + + if not parallel_state.is_pipeline_last_stage(): + return net_output + + x0_pred = batch_mul(c_skip, xt) + batch_mul(c_out, net_output) + + # get noise prediction based on sde + eps_pred = batch_mul(xt - x0_pred, 1.0 / sigma) + + return x0_pred, eps_pred + + def compute_loss_with_epsilon_and_sigma( + self, + data_batch: dict[str, torch.Tensor], + x0_from_data_batch: torch.Tensor, + x0: torch.Tensor, + condition: dict[str, torch.Tensor], + epsilon: torch.Tensor, + sigma: torch.Tensor, + ): + """ + Computes the loss for training. + + Args: + data_batch: Batch of input data. + x0_from_data_batch: Raw input tensor. + x0: Latent tensor. + condition: Conditional input data. + epsilon: Noise tensor. + sigma: Noise level tensor. + + Returns: + The computed loss. + """ + # Get the mean and stand deviation of the marginal probability distribution. + mean, std = self.sde.marginal_prob(x0, sigma) + # Generate noisy observations + xt = mean + batch_mul(std, epsilon) # corrupted data + + if parallel_state.is_pipeline_last_stage(): + # make prediction + x0_pred, eps_pred = self.denoise(xt, sigma, condition) + # loss weights for different noise levels + weights_per_sigma = self.get_per_sigma_loss_weights(sigma=sigma) + pred_mse = (x0 - x0_pred) ** 2 + edm_loss = batch_mul(pred_mse, weights_per_sigma) + + output_batch = { + "x0": x0, + "xt": xt, + "sigma": sigma, + "weights_per_sigma": weights_per_sigma, + "condition": condition, + "model_pred": {"x0_pred": x0_pred, "eps_pred": eps_pred}, + "mse_loss": pred_mse.mean(), + "edm_loss": edm_loss.mean(), + } + return output_batch, pred_mse, edm_loss + else: + # make prediction + x0_pred = self.denoise(xt, sigma, condition) + return x0_pred.contiguous() + + def get_per_sigma_loss_weights(self, sigma: torch.Tensor): + """ + Args: + sigma (tensor): noise level + + Returns: + loss weights per sigma noise level + """ + return (sigma**2 + self.sigma_data**2) / (sigma * self.sigma_data) ** 2 + + def get_condition_uncondition(self, data_batch: Dict): + """Returns conditioning and unconditioning for classifier-free guidance.""" + _, _, condition = self.get_data_and_condition(data_batch, dropout_rate=0.0) + + if "neg_t5_text_embeddings" in data_batch: + data_batch["t5_text_embeddings"] = data_batch["neg_t5_text_embeddings"] + data_batch["t5_text_mask"] = data_batch["neg_t5_text_mask"] + _, _, uncondition = self.get_data_and_condition(data_batch, dropout_rate=1.0) + else: + _, _, uncondition = self.get_data_and_condition(data_batch, dropout_rate=1.0) + + return condition, uncondition + + def get_x0_fn_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + ) -> Callable: + """ + Creates a function to generate denoised predictions with the sampler. + + Args: + data_batch: Batch of input data. + guidance: Guidance scale factor. + is_negative_prompt: Whether to use negative prompts. + + Returns: + A callable to predict clean data (x0). + """ + condition, uncondition = self.get_condition_uncondition(data_batch) + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + cond_x0, _ = self.denoise(noise_x, sigma, condition) + uncond_x0, _ = self.denoise(noise_x, sigma, uncondition) + return cond_x0 + guidance * (cond_x0 - uncond_x0) + + return x0_fn + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + state_shape: Tuple | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + ) -> Tensor: + """ + Generates samples based on input data batch. + + Args: + data_batch: Batch of input data. + guidance: Guidance scale factor. + state_shape: Shape of the state. + is_negative_prompt: Whether to use negative prompts. + num_steps: Number of steps for sampling. + solver_option: SDE Solver option. + + Returns: + Generated samples from diffusion model. + """ + cp_enabled = parallel_state.get_context_parallel_world_size() > 1 + + if self._noise_generator is None: + self._initialize_generators() + x0_fn = self.get_x0_fn_from_batch(data_batch, guidance, is_negative_prompt=is_negative_prompt) + + state_shape = list(state_shape) + state_shape[1] //= parallel_state.get_context_parallel_world_size() + x_sigma_max = ( + torch.randn(state_shape, **self.tensor_kwargs, generator=self._noise_generator) * self.sde.sigma_max + ) + + samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=self.sde.sigma_max) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + samples = cat_outputs_cp(samples, seq_dim=2, cp_group=cp_group) + + return samples + + def draw_training_sigma_and_epsilon(self, x0_size: int, condition: Any) -> torch.Tensor: + """ + Draws training noise (epsilon) and noise levels (sigma). + + Args: + x0_size: Shape of the input tensor. + condition: Conditional input (unused). + + Returns: + Noise level (sigma) and noise (epsilon). + """ + del condition + batch_size = x0_size[0] + if self._noise_generator is None: + self._initialize_generators() + epsilon = torch.randn(x0_size, **self.tensor_kwargs, generator=self._noise_generator) + return self.sde.sample_t(batch_size).to(**self.tensor_kwargs), epsilon + + def random_dropout_input(self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None) -> torch.Tensor: + """ + Applies random dropout to the input tensor. + + Args: + in_tensor: Input tensor. + dropout_rate: Dropout probability (optional). + + Returns: + Conditioning with random dropout applied. + """ + dropout_rate = dropout_rate if dropout_rate is not None else self.dropout_rate + return batch_mul( + torch.bernoulli((1.0 - dropout_rate) * torch.ones(in_tensor.shape[0])).type_as(in_tensor), + in_tensor, + ) + + def get_data_and_condition(self, data_batch: dict[str, Tensor], dropout_rate=0.2) -> Tuple[Tensor]: + """ + Retrieves data and conditioning for model input. + + Args: + data_batch: Batch of input data. + dropout_rate: Dropout probability for conditioning. + + Returns: + Raw data, latent data, and conditioning information. + """ + # Latent state + raw_state = data_batch["video"] * self.sigma_data + # assume data is already encoded + latent_state = raw_state + + # Condition + data_batch["crossattn_emb"] = self.random_dropout_input( + data_batch["t5_text_embeddings"], dropout_rate=dropout_rate + ) + + return raw_state, latent_state, data_batch diff --git a/nemo_vfm/diffusion/sampler/flow_matching/__init__.py b/nemo_vfm/diffusion/sampler/flow_matching/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/flow_matching/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/sampler/flow_matching/flow_match_euler_discrete.py b/nemo_vfm/diffusion/sampler/flow_matching/flow_match_euler_discrete.py new file mode 100644 index 00000000..f7150c24 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/flow_matching/flow_match_euler_discrete.py @@ -0,0 +1,283 @@ +# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from abc import ABC +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + + +class FlowMatchEulerDiscreteScheduler(ABC): + """ + Euler scheduler. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + shift (`float`, defaults to 1.0): + The shift value for the timestep schedule. + """ + + _compatibles = [] + order = 1 + + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + use_dynamic_shifting=False, + base_shift: Optional[float] = 0.5, + max_shift: Optional[float] = 1.15, + base_image_seq_len: Optional[int] = 256, + max_image_seq_len: Optional[int] = 4096, + ): + timesteps = np.linspace(1, num_train_timesteps, num_train_timesteps, dtype=np.float32)[::-1].copy() + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32) + + sigmas = timesteps / num_train_timesteps + if not use_dynamic_shifting: + # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + + self.timesteps = sigmas * num_train_timesteps + + self._step_index = None + self._begin_index = None + + self.sigmas = sigmas.to("cpu") # to avoid too much CPU/GPU communication + self.sigma_min = self.sigmas[-1].item() + self.sigma_max = self.sigmas[0].item() + + self.base_shift = base_shift + self.max_shift = max_shift + self.base_image_seq_len = base_image_seq_len + self.max_image_seq_len = max_image_seq_len + self.use_dynamic_shifting = use_dynamic_shifting + self.num_train_timesteps = num_train_timesteps + self.shift = shift + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_noise( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + """ + Forward process in flow-matching + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype) + + if sample.device.type == "mps" and torch.is_floating_point(timestep): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32) + timestep = timestep.to(sample.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(sample.device) + timestep = timestep.to(sample.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timestep] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timestep.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timestep.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(sample.shape): + sigma = sigma.unsqueeze(-1) + + sample = sigma * noise + (1.0 - sigma) * sample + + return sample + + def _sigma_to_t(self, sigma): + return sigma * self.num_train_timesteps + + def time_shift(self, mu: float, sigma: float, t: torch.Tensor): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + + def set_timesteps( + self, + num_inference_steps: int = None, + device: Union[str, torch.device] = None, + sigmas: Optional[List[float]] = None, + mu: Optional[float] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + + if self.use_dynamic_shifting and mu is None: + raise ValueError(" you have a pass a value for `mu` when `use_dynamic_shifting` is set to be `True`") + + if sigmas is None: + self.num_inference_steps = num_inference_steps + timesteps = np.linspace( + self._sigma_to_t(self.sigma_max), self._sigma_to_t(self.sigma_min), num_inference_steps + ) + + sigmas = timesteps / self.num_train_timesteps + + if self.use_dynamic_shifting: + sigmas = self.time_shift(mu, 1.0, sigmas) + else: + sigmas = self.shift * sigmas / (1 + (self.shift - 1) * sigmas) + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device) + timesteps = sigmas * self.num_train_timesteps + + self.timesteps = timesteps.to(device=device) + self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + + self._step_index = None + self._begin_index = None + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + s_churn: float = 0.0, + s_tmin: float = 0.0, + s_tmax: float = float("inf"), + s_noise: float = 1.0, + generator: Optional[torch.Generator] = None, + ) -> Tuple: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + s_churn (`float`): + s_tmin (`float`): + s_tmax (`float`): + s_noise (`float`, defaults to 1.0): + Scaling factor for noise added to the sample. + generator (`torch.Generator`, *optional*): + A random number generator. + + Returns: + A tuple is returned where the first element is the sample tensor. + """ + + if ( + isinstance(timestep, int) + or isinstance(timestep, torch.IntTensor) + or isinstance(timestep, torch.LongTensor) + ): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + sigma = self.sigmas[self.step_index] + sigma_next = self.sigmas[self.step_index + 1] + prev_sample = sample + (sigma_next - sigma) * model_output + + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + return (prev_sample,) + + def __len__(self): + return self.num_train_timesteps diff --git a/nemo_vfm/diffusion/sampler/res/__init__.py b/nemo_vfm/diffusion/sampler/res/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/diffusion/sampler/res/multi_step.py b/nemo_vfm/diffusion/sampler/res/multi_step.py new file mode 100644 index 00000000..94f920b0 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/res/multi_step.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0116 + +""" +Implementation of multistep methods to solve the ODE in the diffusion model. +""" + +from typing import Callable, List, Tuple + +import torch +from nemo.collections.diffusion.sampler.res.runge_kutta import reg_x0_euler_step, res_x0_rk2_step + + +def order2_fn( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_s: torch.Tensor, x0_preds: torch.Tensor +) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """ + Implementation of the second order multistep method in https://arxiv.org/pdf/2308.02157 + Adams Bashforth approach! + """ + if x0_preds: + x0_s1, s1 = x0_preds[0] + x_t = res_x0_rk2_step(x_s, t, s, x0_s, s1, x0_s1) + else: + x_t = reg_x0_euler_step(x_s, s, t, x0_s)[0] + return x_t, [(x0_s, s)] + + +# key: method name, value: method function +# key: order + algorithm name +MULTISTEP_FNs = { + "2ab": order2_fn, +} + + +def get_multi_step_fn(name: str) -> Callable: + if name in MULTISTEP_FNs: + return MULTISTEP_FNs[name] + methods = "\n\t".join(MULTISTEP_FNs.keys()) + raise RuntimeError("Only support multistep method\n" + methods) + + +def is_multi_step_fn_supported(name: str) -> bool: + """ + Check if the multistep method is supported. + """ + return name in MULTISTEP_FNs diff --git a/nemo_vfm/diffusion/sampler/res/res_sampler.py b/nemo_vfm/diffusion/sampler/res/res_sampler.py new file mode 100644 index 00000000..b2a0413c --- /dev/null +++ b/nemo_vfm/diffusion/sampler/res/res_sampler.py @@ -0,0 +1,277 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116 + +""" +Refined Exponential Solver (RES) in https://arxiv.org/pdf/2308.02157 +""" + +import math +from dataclasses import dataclass +from typing import Any, Callable, List, Literal, Optional, Tuple, Union + +import attrs +import torch +from nemo.collections.diffusion.sampler.res.multi_step import get_multi_step_fn, is_multi_step_fn_supported +from nemo.collections.diffusion.sampler.res.runge_kutta import get_runge_kutta_fn, is_runge_kutta_fn_supported +from tqdm import tqdm + + +COMMON_SOLVER_OPTIONS = Literal["2ab", "2mid", "1euler"] + + +@attrs.define(slots=False) +class SolverConfig: + is_multi: bool = False + rk: str = "2mid" + multistep: str = "2ab" + # following parameters control stochasticity, see EDM paper + # BY default, we use deterministic with no stochasticity + s_churn: float = 0.0 + s_t_max: float = float("inf") + s_t_min: float = 0.05 + s_noise: float = 1.0 + + +@attrs.define(slots=False) +class SolverTimestampConfig: + nfe: int = 50 + t_min: float = 0.002 + t_max: float = 80.0 + order: float = 7.0 + is_forward: bool = False # whether generate forward or backward timestamps + + +@dataclass +class SamplerConfig: + solver: SolverConfig = attrs.field(factory=SolverConfig) + timestamps: SolverTimestampConfig = attrs.field(factory=SolverTimestampConfig) + sample_clean: bool = True # whether run one last step to generate clean image + + +def get_rev_ts( + t_min: float, t_max: float, num_steps: int, ts_order: Union[int, float], is_forward: bool = False +) -> torch.Tensor: + """ + Generate a sequence of reverse time steps. + + Args: + t_min (float): The minimum time value. + t_max (float): The maximum time value. + num_steps (int): The number of time steps to generate. + ts_order (Union[int, float]): The order of the time step progression. + is_forward (bool, optional): If True, returns the sequence in forward order. Defaults to False. + + Returns: + torch.Tensor: A tensor containing the generated time steps in reverse or forward order. + + Raises: + ValueError: If `t_min` is not less than `t_max`. + TypeError: If `ts_order` is not an integer or float. + """ + if t_min >= t_max: + raise ValueError("t_min must be less than t_max") + + if not isinstance(ts_order, (int, float)): + raise TypeError("ts_order must be an integer or float") + + step_indices = torch.arange(num_steps + 1, dtype=torch.float64) + time_steps = ( + t_max ** (1 / ts_order) + step_indices / num_steps * (t_min ** (1 / ts_order) - t_max ** (1 / ts_order)) + ) ** ts_order + + if is_forward: + return time_steps.flip(dims=(0,)) + + return time_steps + + +class RESSampler(torch.nn.Module): + def __init__(self, cfg: Optional[SamplerConfig] = None): + super().__init__() + if cfg is None: + cfg = SamplerConfig() + self.cfg = cfg + + @torch.no_grad() + def forward( + self, + x0_fn: Callable, + x_sigma_max: torch.Tensor, + num_steps: int = 35, + sigma_min: float = 0.002, + sigma_max: float = 80, + rho: float = 7, + S_churn: float = 0, + S_min: float = 0, + S_max: float = float("inf"), + S_noise: float = 1, + solver_option: str = "2ab", + ) -> torch.Tensor: + in_dtype = x_sigma_max.dtype + + def float64_x0_fn(x_B_StateShape: torch.Tensor, t_B: torch.Tensor) -> torch.Tensor: + return x0_fn(x_B_StateShape.to(in_dtype), t_B.to(in_dtype)).to(torch.float64) + + is_multistep = is_multi_step_fn_supported(solver_option) + is_rk = is_runge_kutta_fn_supported(solver_option) + assert is_multistep or is_rk, f"Only support multistep or Runge-Kutta method, got {solver_option}" + + solver_cfg = SolverConfig( + s_churn=S_churn, + s_t_max=S_max, + s_t_min=S_min, + s_noise=S_noise, + is_multi=is_multistep, + rk=solver_option, + multistep=solver_option, + ) + timestamps_cfg = SolverTimestampConfig(nfe=num_steps, t_min=sigma_min, t_max=sigma_max, order=rho) + sampler_cfg = SamplerConfig(solver=solver_cfg, timestamps=timestamps_cfg, sample_clean=True) + + return self._forward_impl(float64_x0_fn, x_sigma_max, sampler_cfg).to(in_dtype) + + @torch.no_grad() + def _forward_impl( + self, + denoiser_fn: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], + noisy_input_B_StateShape: torch.Tensor, + sampler_cfg: Optional[SamplerConfig] = None, + callback_fns: Optional[List[Callable]] = None, + ) -> torch.Tensor: + """ + Internal implementation of the forward pass. + + Args: + denoiser_fn: Function to denoise the input. + noisy_input_B_StateShape: Input tensor with noise. + sampler_cfg: Configuration for the sampler. + callback_fns: List of callback functions to be called during sampling. + + Returns: + torch.Tensor: Denoised output tensor. + """ + sampler_cfg = self.cfg if sampler_cfg is None else sampler_cfg + solver_order = 1 if sampler_cfg.solver.is_multi else int(sampler_cfg.solver.rk[0]) + num_timestamps = sampler_cfg.timestamps.nfe // solver_order + + sigmas_L = get_rev_ts( + sampler_cfg.timestamps.t_min, sampler_cfg.timestamps.t_max, num_timestamps, sampler_cfg.timestamps.order + ).to(noisy_input_B_StateShape.device) + + denoised_output = differential_equation_solver( + denoiser_fn, sigmas_L, sampler_cfg.solver, callback_fns=callback_fns + )(noisy_input_B_StateShape) + + if sampler_cfg.sample_clean: + # Override denoised_output with fully denoised version + ones = torch.ones(denoised_output.size(0), device=denoised_output.device, dtype=denoised_output.dtype) + denoised_output = denoiser_fn(denoised_output, sigmas_L[-1] * ones) + + return denoised_output + + +def fori_loop(lower: int, upper: int, body_fun: Callable[[int, Any], Any], init_val: Any) -> Any: + """ + Implements a for loop with a function. + + Args: + lower: Lower bound of the loop (inclusive). + upper: Upper bound of the loop (exclusive). + body_fun: Function to be applied in each iteration. + init_val: Initial value for the loop. + + Returns: + The final result after all iterations. + """ + val = init_val + for i in tqdm(range(lower, upper)): + val = body_fun(i, val) + return val + + +def differential_equation_solver( + x0_fn: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], + sigmas_L: torch.Tensor, + solver_cfg: SolverConfig, + callback_fns: Optional[List[Callable]] = None, +) -> Callable[[torch.Tensor], torch.Tensor]: + """ + Creates a differential equation solver function. + + Args: + x0_fn: Function to compute x0 prediction. + sigmas_L: Tensor of sigma values with shape [L,]. + solver_cfg: Configuration for the solver. + callback_fns: Optional list of callback functions. + + Returns: + A function that solves the differential equation. + """ + num_step = len(sigmas_L) - 1 + + if solver_cfg.is_multi: + update_step_fn = get_multi_step_fn(solver_cfg.multistep) + else: + update_step_fn = get_runge_kutta_fn(solver_cfg.rk) + + eta = min(solver_cfg.s_churn / (num_step + 1), math.sqrt(1.2) - 1) + + def sample_fn(input_xT_B_StateShape: torch.Tensor) -> torch.Tensor: + """ + Samples from the differential equation. + + Args: + input_xT_B_StateShape: Input tensor with shape [B, StateShape]. + + Returns: + Output tensor with shape [B, StateShape]. + """ + ones_B = torch.ones(input_xT_B_StateShape.size(0), device=input_xT_B_StateShape.device, dtype=torch.float64) + + def step_fn( + i_th: int, state: Tuple[torch.Tensor, Optional[List[torch.Tensor]]] + ) -> Tuple[torch.Tensor, Optional[List[torch.Tensor]]]: + input_x_B_StateShape, x0_preds = state + sigma_cur_0, sigma_next_0 = sigmas_L[i_th], sigmas_L[i_th + 1] + + # algorithm 2: line 4-6 + if solver_cfg.s_t_min < sigma_cur_0 < solver_cfg.s_t_max: + hat_sigma_cur_0 = sigma_cur_0 + eta * sigma_cur_0 + input_x_B_StateShape = input_x_B_StateShape + ( + hat_sigma_cur_0**2 - sigma_cur_0**2 + ).sqrt() * solver_cfg.s_noise * torch.randn_like(input_x_B_StateShape) + sigma_cur_0 = hat_sigma_cur_0 + + if solver_cfg.is_multi: + x0_pred_B_StateShape = x0_fn(input_x_B_StateShape, sigma_cur_0 * ones_B) + output_x_B_StateShape, x0_preds = update_step_fn( + input_x_B_StateShape, sigma_cur_0 * ones_B, sigma_next_0 * ones_B, x0_pred_B_StateShape, x0_preds + ) + else: + output_x_B_StateShape, x0_preds = update_step_fn( + input_x_B_StateShape, sigma_cur_0 * ones_B, sigma_next_0 * ones_B, x0_fn + ) + + if callback_fns: + for callback_fn in callback_fns: + callback_fn(**locals()) + + return output_x_B_StateShape, x0_preds + + x_at_eps, _ = fori_loop(0, num_step, step_fn, [input_xT_B_StateShape, None]) + return x_at_eps + + return sample_fn diff --git a/nemo_vfm/diffusion/sampler/res/runge_kutta.py b/nemo_vfm/diffusion/sampler/res/runge_kutta.py new file mode 100644 index 00000000..015bdcc3 --- /dev/null +++ b/nemo_vfm/diffusion/sampler/res/runge_kutta.py @@ -0,0 +1,331 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Tuple + +import torch +from nemo.collections.diffusion.sampler.batch_ops import batch_mul + + +def phi1(t: torch.Tensor) -> torch.Tensor: + """ + Compute the first order phi function: (exp(t) - 1) / t. + + Args: + t: Input tensor. + + Returns: + Tensor: Result of phi1 function. + """ + input_dtype = t.dtype + t = t.to(dtype=torch.float64) + return (torch.expm1(t) / t).to(dtype=input_dtype) + + +def phi2(t: torch.Tensor) -> torch.Tensor: + """ + Compute the second order phi function: (phi1(t) - 1) / t. + + Args: + t: Input tensor. + + Returns: + Tensor: Result of phi2 function. + """ + input_dtype = t.dtype + t = t.to(dtype=torch.float64) + return ((phi1(t) - 1.0) / t).to(dtype=input_dtype) + + +def res_x0_rk2_step( + x_s: torch.Tensor, + t: torch.Tensor, + s: torch.Tensor, + x0_s: torch.Tensor, + s1: torch.Tensor, + x0_s1: torch.Tensor, +) -> torch.Tensor: + """ + Perform a residual-based 2nd order Runge-Kutta step. + + Args: + x_s: Current state tensor. + t: Target time tensor. + s: Current time tensor. + x0_s: Prediction at current time. + s1: Intermediate time tensor. + x0_s1: Prediction at intermediate time. + + Returns: + Tensor: Updated state tensor. + + Raises: + AssertionError: If step size is too small. + """ + s = -torch.log(s) + t = -torch.log(t) + m = -torch.log(s1) + + dt = t - s + assert not torch.any(torch.isclose(dt, torch.zeros_like(dt), atol=1e-6)), "Step size is too small" + assert not torch.any(torch.isclose(m - s, torch.zeros_like(dt), atol=1e-6)), "Step size is too small" + + c2 = (m - s) / dt + phi1_val, phi2_val = phi1(-dt), phi2(-dt) + + # Handle edge case where t = s = m + b1 = torch.nan_to_num(phi1_val - 1.0 / c2 * phi2_val, nan=0.0) + b2 = torch.nan_to_num(1.0 / c2 * phi2_val, nan=0.0) + + return batch_mul(torch.exp(-dt), x_s) + batch_mul(dt, batch_mul(b1, x0_s) + batch_mul(b2, x0_s1)) + + +def reg_x0_euler_step( + x_s: torch.Tensor, + s: torch.Tensor, + t: torch.Tensor, + x0_s: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a regularized Euler step based on x0 prediction. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_s: Prediction at current time. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current prediction. + """ + coef_x0 = (s - t) / s + coef_xs = t / s + return coef_x0 * x0_s + coef_xs * x_s, x0_s + + +def reg_eps_euler_step( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, eps_s: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a regularized Euler step based on epsilon prediction. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + eps_s: Epsilon prediction at current time. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current x0 prediction. + """ + return x_s + batch_mul(eps_s, t - s), x_s + batch_mul(eps_s, 0 - s) + + +def rk1_euler( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a first-order Runge-Kutta (Euler) step. + + Recommended for diffusion models with guidance or model undertrained + Usually more stable at the cost of a bit slower convergence. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and x0 prediction. + """ + x0_s = x0_fn(x_s, s) + return reg_x0_euler_step(x_s, s, t, x0_s) + + +def rk2_mid_stable( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a stable second-order Runge-Kutta (midpoint) step. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and x0 prediction. + """ + s1 = torch.sqrt(s * t) + x_s1, _ = rk1_euler(x_s, s, s1, x0_fn) + + x0_s1 = x0_fn(x_s1, s1) + return reg_x0_euler_step(x_s, s, t, x0_s1) + + +def rk2_mid(x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a second-order Runge-Kutta (midpoint) step. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and x0 prediction. + """ + s1 = torch.sqrt(s * t) + x_s1, x0_s = rk1_euler(x_s, s, s1, x0_fn) + + x0_s1 = x0_fn(x_s1, s1) + + return res_x0_rk2_step(x_s, t, s, x0_s, s1, x0_s1), x0_s1 + + +def rk_2heun_naive( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a naive second-order Runge-Kutta (Heun's method) step. + Impl based on rho-rk-deis solvers, https://github.com/qsh-zh/deis + Recommended for diffusion models without guidance and relative large NFE + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current state. + """ + x_t, x0_s = rk1_euler(x_s, s, t, x0_fn) + eps_s = batch_mul(1.0 / s, x_t - x0_s) + x0_t = x0_fn(x_t, t) + eps_t = batch_mul(1.0 / t, x_t - x0_t) + + avg_eps = (eps_s + eps_t) / 2 + + return reg_eps_euler_step(x_s, s, t, avg_eps) + + +def rk_2heun_edm( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a naive second-order Runge-Kutta (Heun's method) step. + Impl based no EDM second order Heun method + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current state. + """ + x_t, x0_s = rk1_euler(x_s, s, t, x0_fn) + x0_t = x0_fn(x_t, t) + + avg_x0 = (x0_s + x0_t) / 2 + + return reg_x0_euler_step(x_s, s, t, avg_x0) + + +def rk_3kutta_naive( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a naive third-order Runge-Kutta step. + Impl based on rho-rk-deis solvers, https://github.com/qsh-zh/deis + Recommended for diffusion models without guidance and relative large NFE + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current state. + """ + c2, c3 = 0.5, 1.0 + a31, a32 = -1.0, 2.0 + b1, b2, b3 = 1.0 / 6, 4.0 / 6, 1.0 / 6 + + delta = t - s + + s1 = c2 * delta + s + s2 = c3 * delta + s + x_s1, x0_s = rk1_euler(x_s, s, s1, x0_fn) + eps_s = batch_mul(1.0 / s, x_s - x0_s) + x0_s1 = x0_fn(x_s1, s1) + eps_s1 = batch_mul(1.0 / s1, x_s1 - x0_s1) + + _eps = a31 * eps_s + a32 * eps_s1 + x_s2, _ = reg_eps_euler_step(x_s, s, s2, _eps) + + x0_s2 = x0_fn(x_s2, s2) + eps_s2 = batch_mul(1.0 / s2, x_s2 - x0_s2) + + avg_eps = b1 * eps_s + b2 * eps_s1 + b3 * eps_s2 + return reg_eps_euler_step(x_s, s, t, avg_eps) + + +# key : order + name +RK_FNs = { + "1euler": rk1_euler, + "2mid": rk2_mid, + "2mid_stable": rk2_mid_stable, + "2heun_edm": rk_2heun_edm, + "2heun_naive": rk_2heun_naive, + "3kutta_naive": rk_3kutta_naive, +} + + +def get_runge_kutta_fn(name: str) -> Callable: + """ + Get the specified Runge-Kutta function. + + Args: + name: Name of the Runge-Kutta method. + + Returns: + Callable: The specified Runge-Kutta function. + + Raises: + RuntimeError: If the specified method is not supported. + """ + if name in RK_FNs: + return RK_FNs[name] + methods = "\n\t".join(RK_FNs.keys()) + raise RuntimeError(f"Only support the following Runge-Kutta methods:\n\t{methods}") + + +def is_runge_kutta_fn_supported(name: str) -> bool: + """ + Check if the specified Runge-Kutta function is supported. + + Args: + name: Name of the Runge-Kutta method. + + Returns: + bool: True if the method is supported, False otherwise. + """ + return name in RK_FNs diff --git a/nemo_vfm/diffusion/train.py b/nemo_vfm/diffusion/train.py new file mode 100644 index 00000000..c9a5cdae --- /dev/null +++ b/nemo_vfm/diffusion/train.py @@ -0,0 +1,431 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0116,C0301 + +import os + +import nemo_run as run +import pytorch_lightning as pl +import torch +from megatron.core.distributed import DistributedDataParallelConfig +from megatron.core.optimizer import OptimizerConfig +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.data.diffusion_taskencoder import BasicDiffusionTaskEncoder +from nemo.collections.diffusion.datamodule import DiTActionDataModule, DiTCameraCtrlDataModule, DiTDataModule +from nemo.collections.diffusion.models.model import ( + DiT7BConfig, + DiT7BVideo2WorldActionConfig, + DiT14BVideo2WorldActionConfig, + DiTConfig, + DiTLConfig, + DiTLlama5BConfig, + DiTLlama30BConfig, + DiTModel, + DiTXLConfig, +) +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.callbacks.model_transform import ModelTransform +from nemo.lightning.pytorch.optim import WarmupAnnealingScheduler +from nemo.lightning.pytorch.strategies.utils import RestoreConfig +from pytorch_lightning.loggers import WandbLogger + + +@run.cli.factory +@run.autoconvert +def videofolder_datamodule() -> pl.LightningDataModule: + data_module = DiTDataModule( + seq_length=21760, + micro_batch_size=1, + global_batch_size=1, + num_val_samples=1, + ) + return data_module + + +@run.cli.factory +@run.autoconvert +def videofolder_actiondatamodule() -> pl.LightningDataModule: + data_module = DiTActionDataModule( + seq_length=21760, + micro_batch_size=1, + global_batch_size=int(os.getenv("GLOBAL_BATCH_SIZE", 1)), + num_val_samples=1, + dtype=torch.bfloat16, + ) + return data_module + + +@run.cli.factory +@run.autoconvert +def videofolder_cameractrldatamodule() -> pl.LightningDataModule: + data_module = DiTCameraCtrlDataModule(seq_length=21760, micro_batch_size=1, global_batch_size=1, num_val_samples=1) + return data_module + + +@run.cli.factory +@run.autoconvert +def multimodal_datamodule() -> pl.LightningDataModule: + from nemo.collections.diffusion.data.diffusion_energon_datamodule import DiffusionDataModule + + data_module = DiffusionDataModule( + seq_length=2048, + task_encoder=run.Config(BasicDiffusionTaskEncoder, seq_length=2048), + micro_batch_size=1, + global_batch_size=32, + ) + return data_module + + +@run.cli.factory +@run.autoconvert +def peft(args) -> ModelTransform: + return llm.peft.LoRA( + target_modules=["linear_qkv", "linear_proj"], # , 'linear_fc1', 'linear_fc2'], + dim=args.lora_dim, + ) + + +@run.cli.factory(target=llm.train) +def pretrain() -> run.Partial: + return run.Partial( + llm.train, + model=run.Config( + DiTModel, + config=run.Config(DiTConfig), + ), + data=multimodal_datamodule(), + trainer=run.Config( + nl.Trainer, + devices="auto", + num_nodes=int(os.environ.get("SLURM_NNODES", 1)), + accelerator="gpu", + strategy=run.Config( + nl.MegatronStrategy, + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + pipeline_dtype=torch.bfloat16, + ddp=run.Config( + DistributedDataParallelConfig, + check_for_nan_in_grad=True, + grad_reduce_in_fp32=True, + overlap_grad_reduce=True, + overlap_param_gather=True, + ), + save_ckpt_format="zarr", + ), + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + num_sanity_val_steps=0, + limit_val_batches=0, + max_epochs=10000, + log_every_n_steps=1, + callbacks=[ + run.Config( + ModelCheckpoint, + monitor="reduced_train_loss", + # Required filename format to support custom gather during training. + filename="{epoch}-{step}", + every_n_train_steps=200, + save_top_k=100, + save_weights_only=True, + ), + run.Config(PreemptionCallback), + ], + ), + log=nl.NeMoLogger(wandb=(WandbLogger() if "WANDB_API_KEY" in os.environ else None)), + optim=run.Config( + nl.MegatronOptimizerModule, + config=run.Config( + OptimizerConfig, + lr=1e-4, + bf16=True, + params_dtype=torch.bfloat16, + use_distributed_optimizer=True, + weight_decay=0, + ), + ), + tokenizer=None, + resume=run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + ), + model_transform=None, + ) + + +@run.cli.factory(target=llm.train) +def pretrain_xl() -> run.Partial: + recipe = pretrain() + recipe.model.config = run.Config(DiTXLConfig) + return recipe + + +@run.cli.factory(target=llm.train) +def pretrain_l() -> run.Partial: + recipe = pretrain() + recipe.model.config = run.Config(DiTLConfig) + return recipe + + +@run.cli.factory(target=llm.train) +def pretrain_7b() -> run.Partial: + recipe = pretrain() + recipe.model.config = run.Config(DiT7BConfig) + recipe.data.global_batch_size = 4608 + recipe.data.micro_batch_size = 9 + recipe.data.num_workers = 15 + recipe.data.use_train_split_for_val = True + recipe.data.seq_length = 260 + recipe.data.task_encoder.seq_length = 260 + recipe.trainer.val_check_interval = 1000 + recipe.log.log_dir = "nemo_experiments/dit7b" + recipe.optim.lr_scheduler = run.Config(nl.lr_scheduler.WarmupHoldPolicyScheduler, warmup_steps=100, hold_steps=1e9) + recipe.optim.config.weight_decay = 0.1 + recipe.optim.config.adam_beta1 = 0.9 + recipe.optim.config.adam_beta2 = 0.95 + + return recipe + + +@run.cli.factory(target=llm.train) +def finetune_7b_action_control() -> run.Partial: + experiment_dir = os.getenv("EXP_DIR", "nemo_experiments/cosmos_diffusion_7b_video2world_action_finetune") + + return run.Partial( + llm.train, + model=run.Config( + DiTModel, + config=run.Config(DiT7BVideo2WorldActionConfig), + ), + data=videofolder_actiondatamodule(), + trainer=run.Config( + nl.Trainer, + devices="auto", + num_nodes=int(os.environ.get("SLURM_NNODES", 1)), + accelerator="gpu", + strategy=run.Config( + nl.MegatronStrategy, + tensor_model_parallel_size=int(os.getenv("TENSOR_MODEL_PARALLEL_SIZE", 8)), + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=(True if int(os.getenv("TENSOR_MODEL_PARALLEL_SIZE", 8)) > 1 else False), + pipeline_dtype=torch.bfloat16, + ddp=run.Config( + DistributedDataParallelConfig, + check_for_nan_in_grad=True, + grad_reduce_in_fp32=True, + overlap_grad_reduce=False, + overlap_param_gather=False, + ), + save_ckpt_format="zarr", + ckpt_load_strictness=False, # Need non-strict checkpoint loading, because the action model has more layers than the pre-trained checkpoint! + ckpt_async_save=False, + ), + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + num_sanity_val_steps=0, + limit_val_batches=0, + max_epochs=10000, + max_steps=10000, + log_every_n_steps=1, + callbacks=[ + run.Config( + ModelCheckpoint, + # Required filename format to support custom gather during training. + filename="{epoch}-{step}", + dirpath=experiment_dir, + monitor="reduced_train_loss", + every_n_train_steps=250, + save_top_k=5, + save_weights_only=True, + ), + run.Config(PreemptionCallback), + ], + ), + log=nl.NeMoLogger( + wandb=( + WandbLogger( + name=os.getenv("WANDB_RUN_NAME", "cosmos-7b-diffusion-action"), + group=os.getenv("WANDB_RUN_GROUP", "nvidia-dir"), + project=os.getenv("WANDB_PROJECT", "cosmos-nemo-diffusion-action-control-gtc"), + ) + if "WANDB_API_KEY" in os.environ + else None + ), + log_dir=experiment_dir, + ), + optim=run.Config( + nl.MegatronOptimizerModule, + config=run.Config( + OptimizerConfig, + lr=1e-6, + bf16=True, + params_dtype=torch.bfloat16, + use_distributed_optimizer=True, + weight_decay=0.1, + ), + lr_scheduler=run.Config( + WarmupAnnealingScheduler, + warmup_steps=2600, + min_lr=4e-10, + max_steps=10000, + ), + ), + tokenizer=None, + resume=run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + ), + model_transform=None, + ) + + +@run.cli.factory(target=llm.train) +def finetune_14b_action_control() -> run.Partial: + experiment_dir = os.getenv("EXP_DIR", "nemo_experiments/cosmos_diffusion_14b_video2world_action_finetune") + + return run.Partial( + llm.train, + model=run.Config( + DiTModel, + config=run.Config(DiT14BVideo2WorldActionConfig), + ), + data=videofolder_actiondatamodule(), + trainer=run.Config( + nl.Trainer, + devices="auto", + num_nodes=int(os.environ.get("SLURM_NNODES", 1)), + accelerator="gpu", + strategy=run.Config( + nl.MegatronStrategy, + tensor_model_parallel_size=int(os.getenv("TENSOR_MODEL_PARALLEL_SIZE", 8)), + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=(True if int(os.getenv("TENSOR_MODEL_PARALLEL_SIZE", 8)) > 1 else False), + pipeline_dtype=torch.bfloat16, + ddp=run.Config( + DistributedDataParallelConfig, + check_for_nan_in_grad=True, + grad_reduce_in_fp32=True, + overlap_grad_reduce=False, + overlap_param_gather=False, + ), + save_ckpt_format="zarr", + ckpt_load_strictness=False, # Need non-strict checkpoint loading, because the action model has more layers than the pre-trained checkpoint! + ckpt_async_save=False, + ), + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + num_sanity_val_steps=0, + limit_val_batches=0, + max_epochs=10000, + max_steps=10000, + log_every_n_steps=1, + callbacks=[ + run.Config( + ModelCheckpoint, + # Required filename format to support custom gather during training. + filename="{epoch}-{step}", + dirpath=experiment_dir, + monitor="reduced_train_loss", + every_n_train_steps=250, + save_top_k=5, + save_weights_only=True, + ), + run.Config(PreemptionCallback), + ], + ), + log=nl.NeMoLogger( + wandb=( + WandbLogger( + name=os.getenv("WANDB_RUN_NAME", "cosmos-14b-diffusion-action"), + group=os.getenv("WANDB_RUN_GROUP", "nvidia-dir"), + project=os.getenv("WANDB_PROJECT", "cosmos-nemo-diffusion-action-control-gtc"), + ) + if "WANDB_API_KEY" in os.environ + else None + ), + log_dir=experiment_dir, + ), + optim=run.Config( + nl.MegatronOptimizerModule, + config=run.Config( + OptimizerConfig, + lr=1e-6, + bf16=True, + params_dtype=torch.bfloat16, + use_distributed_optimizer=True, + weight_decay=0.1, + ), + lr_scheduler=run.Config( + WarmupAnnealingScheduler, + warmup_steps=2600, + min_lr=4e-10, + max_steps=10000, + ), + ), + tokenizer=None, + resume=run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + ), + model_transform=None, + ) + + +@run.cli.factory(target=llm.train) +def pretrain_ditllama5b() -> run.Partial: + recipe = pretrain_7b() + recipe.data.micro_batch_size = 12 + recipe.model.config = run.Config(DiTLlama5BConfig) + recipe.log.log_dir = "nemo_experiments/ditllama5b" + return recipe + + +@run.cli.factory(target=llm.train) +def pretrain_ditllama30b() -> run.Partial: + recipe = pretrain_ditllama5b() + recipe.model.config = run.Config(DiTLlama30BConfig) + recipe.data.global_batch_size = 9216 + recipe.data.micro_batch_size = 6 + recipe.log.log_dir = "nemo_experiments/ditllama30b" + return recipe + + +@run.cli.factory(target=llm.train) +def dreambooth() -> run.Partial: + recipe = pretrain() + recipe.optim.config.lr = 1e-6 + recipe.data = multimodal_datamodule() + recipe.model.config = run.Config(DiT7BConfig) + + recipe.trainer.max_steps = 1000 + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + + recipe.resume.restore_config = run.Config(RestoreConfig) + recipe.resume.resume_if_exists = False + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=dreambooth) diff --git a/nemo_vfm/diffusion/utils/__init__.py b/nemo_vfm/diffusion/utils/__init__.py new file mode 100644 index 00000000..d9155f92 --- /dev/null +++ b/nemo_vfm/diffusion/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/utils/flux_ckpt_converter.py b/nemo_vfm/diffusion/utils/flux_ckpt_converter.py new file mode 100644 index 00000000..4b8ba35f --- /dev/null +++ b/nemo_vfm/diffusion/utils/flux_ckpt_converter.py @@ -0,0 +1,225 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from safetensors.torch import load_file as load_safetensors + + +def _import_qkv_bias(transformer_config, qb, kb, vb): + head_num = transformer_config.num_attention_heads + num_query_groups = transformer_config.num_query_groups + heads_per_group = head_num // num_query_groups + hidden_size = transformer_config.hidden_size + head_num = transformer_config.num_attention_heads + head_size = hidden_size // head_num + + new_q_bias_tensor_shape = (head_num, head_size) + new_kv_bias_tensor_shape = (num_query_groups, head_size) + + qb = qb.view(*new_q_bias_tensor_shape) + kb = kb.view(*new_kv_bias_tensor_shape) + vb = vb.view(*new_kv_bias_tensor_shape) + + qkv_bias_l = [] + for i in range(num_query_groups): + qkv_bias_l.append(qb[i * heads_per_group : (i + 1) * heads_per_group, :]) + qkv_bias_l.append(kb[i : i + 1, :]) + qkv_bias_l.append(vb[i : i + 1, :]) + + qkv_bias = torch.cat(qkv_bias_l) + qkv_bias = qkv_bias.reshape([head_size * (head_num + 2 * num_query_groups)]) + + return qkv_bias + + +def _import_qkv(transformer_config, q, k, v): + head_num = transformer_config.num_attention_heads + num_query_groups = transformer_config.num_query_groups + heads_per_group = head_num // num_query_groups + hidden_size = transformer_config.hidden_size + head_num = transformer_config.num_attention_heads + head_size = hidden_size // head_num + + old_tensor_shape = q.size() + new_q_tensor_shape = (head_num, head_size) + old_tensor_shape[1:] + new_kv_tensor_shape = (num_query_groups, head_size) + old_tensor_shape[1:] + + q = q.view(*new_q_tensor_shape) + k = k.view(*new_kv_tensor_shape) + v = v.view(*new_kv_tensor_shape) + + qkv_weights_l = [] + for i in range(num_query_groups): + qkv_weights_l.append(q[i * heads_per_group : (i + 1) * heads_per_group, :, :]) + qkv_weights_l.append(k[i : i + 1, :, :]) + qkv_weights_l.append(v[i : i + 1, :, :]) + qkv_weights = torch.cat(qkv_weights_l) + assert qkv_weights.ndim == 3, qkv_weights.shape + assert qkv_weights.shape[0] == (heads_per_group + 2) * num_query_groups, qkv_weights.shape + assert qkv_weights.shape[1] == head_size, qkv_weights.shape + assert qkv_weights.shape[2] == old_tensor_shape[1], qkv_weights.shape + + qkv_weights = qkv_weights.reshape([head_size * (head_num + 2 * num_query_groups), hidden_size]) + + return qkv_weights + + +flux_key_mapping = { + "double_blocks": { + "norm1.linear.weight": "adaln.adaLN_modulation.1.weight", + "norm1.linear.bias": "adaln.adaLN_modulation.1.bias", + "norm1_context.linear.weight": "adaln_context.adaLN_modulation.1.weight", + "norm1_context.linear.bias": "adaln_context.adaLN_modulation.1.bias", + "attn.norm_q.weight": "self_attention.q_layernorm.weight", + "attn.norm_k.weight": "self_attention.k_layernorm.weight", + "attn.norm_added_q.weight": "self_attention.added_q_layernorm.weight", + "attn.norm_added_k.weight": "self_attention.added_k_layernorm.weight", + "attn.to_out.0.weight": "self_attention.linear_proj.weight", + "attn.to_out.0.bias": "self_attention.linear_proj.bias", + "attn.to_add_out.weight": "self_attention.added_linear_proj.weight", + "attn.to_add_out.bias": "self_attention.added_linear_proj.bias", + "ff.net.0.proj.weight": "mlp.linear_fc1.weight", + "ff.net.0.proj.bias": "mlp.linear_fc1.bias", + "ff.net.2.weight": "mlp.linear_fc2.weight", + "ff.net.2.bias": "mlp.linear_fc2.bias", + "ff_context.net.0.proj.weight": "context_mlp.linear_fc1.weight", + "ff_context.net.0.proj.bias": "context_mlp.linear_fc1.bias", + "ff_context.net.2.weight": "context_mlp.linear_fc2.weight", + "ff_context.net.2.bias": "context_mlp.linear_fc2.bias", + }, + "single_blocks": { + "norm.linear.weight": "adaln.adaLN_modulation.1.weight", + "norm.linear.bias": "adaln.adaLN_modulation.1.bias", + "proj_mlp.weight": "mlp.linear_fc1.weight", + "proj_mlp.bias": "mlp.linear_fc1.bias", + # 'proj_out.weight': 'proj_out.weight', + # 'proj_out.bias': 'proj_out.bias', + "attn.norm_q.weight": "self_attention.q_layernorm.weight", + "attn.norm_k.weight": "self_attention.k_layernorm.weight", + }, + "norm_out.linear.bias": "norm_out.adaLN_modulation.1.bias", + "norm_out.linear.weight": "norm_out.adaLN_modulation.1.weight", + "proj_out.bias": "proj_out.bias", + "proj_out.weight": "proj_out.weight", + "time_text_embed.guidance_embedder.linear_1.bias": "guidance_embedding.in_layer.bias", + "time_text_embed.guidance_embedder.linear_1.weight": "guidance_embedding.in_layer.weight", + "time_text_embed.guidance_embedder.linear_2.bias": "guidance_embedding.out_layer.bias", + "time_text_embed.guidance_embedder.linear_2.weight": "guidance_embedding.out_layer.weight", + "x_embedder.bias": "img_embed.bias", + "x_embedder.weight": "img_embed.weight", + "time_text_embed.timestep_embedder.linear_1.bias": "timestep_embedding.time_embedder.in_layer.bias", + "time_text_embed.timestep_embedder.linear_1.weight": "timestep_embedding.time_embedder.in_layer.weight", + "time_text_embed.timestep_embedder.linear_2.bias": "timestep_embedding.time_embedder.out_layer.bias", + "time_text_embed.timestep_embedder.linear_2.weight": "timestep_embedding.time_embedder.out_layer.weight", + "context_embedder.bias": "txt_embed.bias", + "context_embedder.weight": "txt_embed.weight", + "time_text_embed.text_embedder.linear_1.bias": "vector_embedding.in_layer.bias", + "time_text_embed.text_embedder.linear_1.weight": "vector_embedding.in_layer.weight", + "time_text_embed.text_embedder.linear_2.bias": "vector_embedding.out_layer.bias", + "time_text_embed.text_embedder.linear_2.weight": "vector_embedding.out_layer.weight", + "controlnet_x_embedder.weight": "controlnet_x_embedder.weight", + "controlnet_x_embedder.bias": "controlnet_x_embedder.bias", +} + + +def flux_transformer_converter(ckpt_path=None, transformer_config=None): + # pylint: disable=C0116 + diffuser_state_dict = {} + if os.path.isdir(ckpt_path): + files = os.listdir(ckpt_path) + for file in files: + if file.endswith(".safetensors"): + loaded_dict = load_safetensors(os.path.join(ckpt_path, file)) + diffuser_state_dict.update(loaded_dict) + elif os.path.isfile(ckpt_path): + diffuser_state_dict = load_safetensors(ckpt_path) + else: + raise FileNotFoundError("Please provide a valid ckpt path.") + new_state_dict = {} + num_single_blocks = -1 + num_double_blocks = -1 + for key, value in diffuser_state_dict.items(): + if "attn.to_q" in key or "attn.to_k" in key or "attn.to_v" in key: + continue + if "attn.add_q_proj" in key or "attn.add_k_proj" in key or "attn.add_v_proj" in key: + continue + if key.startswith("transformer_blocks"): + temp = key.split(".") + idx, k = temp[1], ".".join(temp[2:]) + num_double_blocks = max(int(idx), num_double_blocks) + new_key = ".".join(["double_blocks", idx, flux_key_mapping["double_blocks"][k]]) + elif key.startswith("single_transformer_blocks"): + if "proj_out" in key: + continue + temp = key.split(".") + idx, k = temp[1], ".".join(temp[2:]) + num_single_blocks = max(int(idx), num_single_blocks) + new_key = ".".join(["single_blocks", idx, flux_key_mapping["single_blocks"][k]]) + elif key.startswith("controlnet_blocks"): + new_key = "controlnet_double_blocks." + ".".join(key.split(".")[1:]) + else: + new_key = flux_key_mapping[key] + new_state_dict[new_key] = value + for i in range(num_double_blocks + 1): + new_key = f"double_blocks.{str(i)}.self_attention.linear_qkv.weight" + qk, kk, vk = [f"transformer_blocks.{str(i)}.attn.to_{n}.weight" for n in ("q", "k", "v")] + new_state_dict[new_key] = _import_qkv( + transformer_config, diffuser_state_dict[qk], diffuser_state_dict[kk], diffuser_state_dict[vk] + ) + new_key = f"double_blocks.{str(i)}.self_attention.linear_qkv.bias" + qk, kk, vk = [f"transformer_blocks.{str(i)}.attn.to_{n}.bias" for n in ("q", "k", "v")] + new_state_dict[new_key] = _import_qkv_bias( + transformer_config, diffuser_state_dict[qk], diffuser_state_dict[kk], diffuser_state_dict[vk] + ) + new_key = f"double_blocks.{str(i)}.self_attention.added_linear_qkv.weight" + qk, kk, vk = [f"transformer_blocks.{str(i)}.attn.add_{n}_proj.weight" for n in ("q", "k", "v")] + new_state_dict[new_key] = _import_qkv( + transformer_config, diffuser_state_dict[qk], diffuser_state_dict[kk], diffuser_state_dict[vk] + ) + new_key = f"double_blocks.{str(i)}.self_attention.added_linear_qkv.bias" + qk, kk, vk = [f"transformer_blocks.{str(i)}.attn.add_{n}_proj.bias" for n in ("q", "k", "v")] + new_state_dict[new_key] = _import_qkv_bias( + transformer_config, diffuser_state_dict[qk], diffuser_state_dict[kk], diffuser_state_dict[vk] + ) + + for i in range(num_single_blocks + 1): + new_key = f"single_blocks.{str(i)}.self_attention.linear_qkv.weight" + qk, kk, vk = [f"single_transformer_blocks.{str(i)}.attn.to_{n}.weight" for n in ("q", "k", "v")] + new_state_dict[new_key] = _import_qkv( + transformer_config, diffuser_state_dict[qk], diffuser_state_dict[kk], diffuser_state_dict[vk] + ) + new_key = f"single_blocks.{str(i)}.self_attention.linear_qkv.bias" + qk, kk, vk = [f"single_transformer_blocks.{str(i)}.attn.to_{n}.bias" for n in ("q", "k", "v")] + new_state_dict[new_key] = _import_qkv_bias( + transformer_config, diffuser_state_dict[qk], diffuser_state_dict[kk], diffuser_state_dict[vk] + ) + + ( + new_state_dict[f"single_blocks.{str(i)}.mlp.linear_fc2.weight"], + new_state_dict[f"single_blocks.{str(i)}.self_attention.linear_proj.weight"], + ) = ( + diffuser_state_dict[f"single_transformer_blocks.{str(i)}.proj_out.weight"].detach()[:, 3072:].clone(), + diffuser_state_dict[f"single_transformer_blocks.{str(i)}.proj_out.weight"].detach()[:, :3072].clone(), + ) + + new_state_dict[f"single_blocks.{str(i)}.mlp.linear_fc2.bias"] = ( + diffuser_state_dict[f"single_transformer_blocks.{str(i)}.proj_out.bias"].detach().clone() + ) + new_state_dict[f"single_blocks.{str(i)}.self_attention.linear_proj.bias"] = ( + diffuser_state_dict[f"single_transformer_blocks.{str(i)}.proj_out.bias"].detach().clone() + ) + + return new_state_dict diff --git a/nemo_vfm/diffusion/utils/flux_pipeline_utils.py b/nemo_vfm/diffusion/utils/flux_pipeline_utils.py new file mode 100644 index 00000000..56c1d38e --- /dev/null +++ b/nemo_vfm/diffusion/utils/flux_pipeline_utils.py @@ -0,0 +1,62 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from megatron.core.transformer.utils import openai_gelu +from nemo.collections.diffusion.models.flux.model import ClipConfig, FluxConfig, FluxModelParams, T5Config +from nemo.collections.diffusion.vae.autoencoder import AutoEncoderConfig + + +configs = { + "dev": FluxModelParams( + flux_config=FluxConfig( + num_joint_layers=19, + num_single_layers=38, + hidden_size=3072, + num_attention_heads=24, + activation_func=openai_gelu, + add_qkv_bias=True, + in_channels=64, + context_dim=4096, + model_channels=256, + patch_size=1, + guidance_embed=True, + vec_in_dim=768, + ckpt_path=None, + do_convert_from_hf=False, + ), + vae_config=AutoEncoderConfig( + ch_mult=[1, 2, 4, 4], + attn_resolutions=[], + resolution=256, + in_channels=3, + ch=128, + out_ch=3, + num_res_blocks=2, + z_channels=16, + scale_factor=0.3611, + shift_factor=0.1159, + ckpt=None, + ), + clip_params=ClipConfig( + max_length=77, + always_return_pooled=True, + ), + t5_params=T5Config( + max_length=512, + ), + scheduler_steps=1000, + device="cpu", + ) +} diff --git a/nemo_vfm/diffusion/utils/mcore_parallel_utils.py b/nemo_vfm/diffusion/utils/mcore_parallel_utils.py new file mode 100644 index 00000000..ca1686c9 --- /dev/null +++ b/nemo_vfm/diffusion/utils/mcore_parallel_utils.py @@ -0,0 +1,86 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +Megatron Model Parallel Initialization +""" + +import os + +import megatron.core.parallel_state as ps +import torch + + +# pylint: disable=C0116 +class Utils: + world_size = torch.cuda.device_count() + # rank = int(os.environ["LOCAL_RANK"]) + rank = 0 + + @staticmethod + def initialize_distributed(tensor_model_parallel_size=1, pipeline_model_parallel_size=1, context_parallel_size=1): + ps.destroy_model_parallel() + + # Torch setup for distributed training + rank = int(os.environ["LOCAL_RANK"]) + world_size = 1 # torch.cuda.device_count() + torch.cuda.set_device(rank) + torch.distributed.init_process_group(world_size=world_size, rank=rank) + + # Megatron core distributed training initialization + ps.initialize_model_parallel( + tensor_model_parallel_size, pipeline_model_parallel_size, context_parallel_size=context_parallel_size + ) + + @staticmethod + def set_world_size(world_size=None, rank=None): + Utils.world_size = torch.cuda.device_count() if world_size is None else world_size + if torch.distributed.is_initialized() and Utils.world_size != torch.distributed.get_world_size(): + torch.distributed.destroy_process_group() + + if rank is None: + # Utils.rank = int(os.environ["LOCAL_RANK"]) + Utils.rank = 0 + if Utils.rank >= Utils.world_size: + Utils.rank = -1 + else: + Utils.rank = rank + + @staticmethod + def destroy_model_parallel(): + ps.destroy_model_parallel() + torch.distributed.barrier() + + @staticmethod + def initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + virtual_pipeline_model_parallel_size=None, + pipeline_model_parallel_split_rank=None, + **kwargs, + ): + ps.destroy_model_parallel() + Utils.initialize_distributed() + ps.initialize_model_parallel( + tensor_model_parallel_size, + pipeline_model_parallel_size, + virtual_pipeline_model_parallel_size, + pipeline_model_parallel_split_rank, + **kwargs, + ) + + +# pylint: disable=C0116 diff --git a/nemo_vfm/diffusion/vae/__init__.py b/nemo_vfm/diffusion/vae/__init__.py new file mode 100644 index 00000000..9e325007 --- /dev/null +++ b/nemo_vfm/diffusion/vae/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/diffusion/vae/autoencoder.py b/nemo_vfm/diffusion/vae/autoencoder.py new file mode 100644 index 00000000..53e40658 --- /dev/null +++ b/nemo_vfm/diffusion/vae/autoencoder.py @@ -0,0 +1,340 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from dataclasses import dataclass + +import numpy as np +import torch +from torch import Tensor, nn + +from nemo.collections.diffusion.vae.blocks import Downsample, Normalize, ResnetBlock, Upsample, make_attn + + +# pylint: disable=C0116 +@dataclass +class AutoEncoderConfig: + ch_mult: list[int] + attn_resolutions: list[int] + resolution: int = 256 + in_channels: int = 3 + ch: int = 128 + out_ch: int = 3 + num_res_blocks: int = 2 + z_channels: int = 16 + scale_factor: float = 0.3611 + shift_factor: float = 0.1159 + attn_type: str = "vanilla" + double_z: bool = True + dropout: float = 0.0 + ckpt: str = None + + +def nonlinearity(x): + # swish + return torch.nn.functional.silu(x) + + +class Encoder(nn.Module): + def __init__( + self, + *, + ch: int, + out_ch: int, + ch_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + in_channels: int, + resolution: int, + z_channels: int, + dropout=0.0, + resamp_with_conv=True, + double_z=True, + use_linear_attn=False, + attn_type="vanilla", + ): + super().__init__() + if use_linear_attn: + attn_type = "linear" + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + # downsampling + self.conv_in = torch.nn.Conv2d(in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + curr_res = resolution + in_ch_mult = (1,) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, out_channels=block_out, temb_channels=self.temb_ch, dropout=dropout + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout + ) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout + ) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, 2 * z_channels if double_z else z_channels, kernel_size=3, stride=1, padding=1 + ) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + def __init__( + self, + *, + ch: int, + out_ch: int, + ch_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + in_channels: int, + resolution: int, + z_channels: int, + dropout=0.0, + resamp_with_conv=True, + give_pre_end=False, + tanh_out=False, + use_linear_attn=False, + attn_type="vanilla", + **ignorekwargs, + ): + super().__init__() + if use_linear_attn: + attn_type = "linear" + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + + # compute in_ch_mult, block_in and curr_res at lowest res + in_ch_mult = (1,) + tuple(ch_mult) + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + print("Working with z of shape {} = {} dimensions.".format(self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = torch.nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout + ) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, out_channels=block_in, temb_channels=self.temb_ch, dropout=dropout + ) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, out_channels=block_out, temb_channels=self.temb_ch, dropout=dropout + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z): + # assert z.shape[1:] == self.z_shape[1:] + self.last_z_shape = z.shape + + # timestep embedding + temb = None + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + if self.tanh_out: + h = torch.tanh(h) + return h + + +class DiagonalGaussian(nn.Module): + def __init__(self, sample: bool = True, chunk_dim: int = 1): + super().__init__() + self.sample = sample + self.chunk_dim = chunk_dim + + def forward(self, z: Tensor) -> Tensor: + mean, logvar = torch.chunk(z, 2, dim=self.chunk_dim) + if self.sample: + std = torch.exp(0.5 * logvar) + return mean + std * torch.randn_like(mean) + else: + return mean + + +class AutoEncoder(nn.Module): + def __init__(self, params: AutoEncoderConfig): + super().__init__() + self.encoder = Encoder( + resolution=params.resolution, + in_channels=params.in_channels, + ch=params.ch, + ch_mult=params.ch_mult, + num_res_blocks=params.num_res_blocks, + z_channels=params.z_channels, + double_z=params.double_z, + attn_type=params.attn_type, + dropout=params.dropout, + out_ch=params.out_ch, + attn_resolutions=params.attn_resolutions, + ) + self.decoder = Decoder( + resolution=params.resolution, + in_channels=params.in_channels, + ch=params.ch, + out_ch=params.out_ch, + ch_mult=params.ch_mult, + num_res_blocks=params.num_res_blocks, + z_channels=params.z_channels, + double_z=params.double_z, + attn_type=params.attn_type, + dropout=params.dropout, + attn_resolutions=params.attn_resolutions, + ) + self.reg = DiagonalGaussian() + + self.scale_factor = params.scale_factor + self.shift_factor = params.shift_factor + self.params = params + + if params.ckpt is not None: + self.load_from_checkpoint(params.ckpt) + + def encode(self, x: Tensor) -> Tensor: + z = self.reg(self.encoder(x)) + z = self.scale_factor * (z - self.shift_factor) + return z + + def decode(self, z: Tensor) -> Tensor: + z = z / self.scale_factor + self.shift_factor + return self.decoder(z) + + def forward(self, x: Tensor) -> Tensor: + return self.decode(self.encode(x)) + + def load_from_checkpoint(self, ckpt_path): + from safetensors.torch import load_file as load_sft + + state_dict = load_sft(ckpt_path) + missing, unexpected = self.load_state_dict(state_dict) + if len(missing) > 0: + logger.warning(f"Following keys are missing from checkpoint loaded: {missing}") + + +# flake8: noqa diff --git a/nemo_vfm/diffusion/vae/autovae.py b/nemo_vfm/diffusion/vae/autovae.py new file mode 100644 index 00000000..0546b61c --- /dev/null +++ b/nemo_vfm/diffusion/vae/autovae.py @@ -0,0 +1,321 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import itertools +import time +from typing import Dict, List + +import torch +import torch.profiler +from diffusers import AutoencoderKL +from torch import nn + + +class VAEGenerator: + """ + A class for generating and searching different Variational Autoencoder (VAE) configurations. + + This class provides functionality to generate various VAE architecture configurations + given a specific input resolution and compression ratio. It allows searching through a + design space to find configurations that match given parameter and memory budgets. + """ + + def __init__(self, input_resolution: int = 1024, compression_ratio: int = 16) -> None: + if input_resolution == 1024: + assert compression_ratio in [8, 16] + elif input_resolution == 2048: + assert compression_ratio in [8, 16, 32] + else: + raise NotImplementedError("Higher resolution than 2028 is not implemented yet!") + + self._input_resolution = input_resolution + self._compression_ratio = compression_ratio + + def _generate_input(self): + """ + Generate a random input tensor with the specified input resolution. + + The tensor is placed on the GPU in half-precision (float16). + """ + random_tensor = torch.rand(1, 3, self.input_resolution, self.input_resolution) + random_tensor = random_tensor.to(dtype=torch.float16, device="cuda") + return random_tensor + + def _count_parameters(self, model: nn.Module = None): + """ + Count the number of trainable parameters in a given model. + + Args: + model (nn.Module): The model for which to count parameters. + + Returns: + int: The number of trainable parameters. + """ + assert model is not None, "Please provide a nn.Module to count the parameters." + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + def _load_base_json_skeleton(self): + """ + Load a base configuration skeleton for the VAE. + + Returns: + dict: A dictionary representing the base configuration JSON skeleton. + """ + skeleton = { + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.20.0.dev0", + "_name_or_path": "../sdxl-vae/", + "act_fn": "silu", + "block_out_channels": [], + "down_block_types": [], + "force_upcast": False, + "in_channels": 3, + "latent_channels": -1, # 16 + "layers_per_block": -1, # 2 + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 1024, # resolution size + "scaling_factor": 0.13025, + "up_block_types": [], + } + return skeleton + + def _generate_all_combinations(self, attr): + """ + Generates all possible combinations from a search space dictionary. + + Args: + attr (dict): A dictionary where each key has a list of possible values. + + Returns: + List[Dict]: A list of dictionaries, each representing a unique combination of attributes. + """ + keys = list(attr.keys()) + choices = [attr[key] for key in keys] + all_combinations = list(itertools.product(*choices)) + + combination_dicts = [] + for combination in all_combinations: + combination_dict = {key: value for key, value in zip(keys, combination)} + combination_dicts.append(combination_dict) + + return combination_dicts + + def _assign_attributes(self, choice): + """ + Assign a chosen set of attributes to the base VAE configuration skeleton. + + Args: + choice (dict): A dictionary of attributes to assign to the skeleton. + + Returns: + dict: A dictionary representing the updated VAE configuration. + """ + search_space_skleton = self._load_base_json_skeleton() + search_space_skleton["down_block_types"] = choice["down_block_types"] + search_space_skleton["up_block_types"] = choice["up_block_types"] + search_space_skleton["block_out_channels"] = choice["block_out_channels"] + search_space_skleton["layers_per_block"] = choice["layers_per_block"] + search_space_skleton["latent_channels"] = choice["latent_channels"] + return search_space_skleton + + def _search_space_16x1024(self): + """ + Define the search space for a 16x compression ratio at 1024 resolution. + + Returns: + dict: A dictionary defining lists of possible attribute values. + """ + attr = {} + attr["down_block_types"] = [["DownEncoderBlock2D"] * 5] + attr["up_block_types"] = [["UpDecoderBlock2D"] * 5] + attr["block_out_channels"] = [ + [128, 256, 512, 512, 512], + [128, 256, 512, 512, 1024], + [128, 256, 512, 1024, 2048], + [64, 128, 256, 512, 512], + ] + attr["layers_per_block"] = [1, 2, 3] + attr["latent_channels"] = [4, 16, 32, 64] + return attr + + def _search_space_8x1024(self): + """ + Define the search space for an 8x compression ratio at 1024 resolution. + + Returns: + dict: A dictionary defining lists of possible attribute values. + """ + attr = {} + attr["down_block_types"] = [["DownEncoderBlock2D"] * 4] + attr["up_block_types"] = [["UpDecoderBlock2D"] * 4] + attr["block_out_channels"] = [[128, 256, 512, 512], [128, 256, 512, 1024], [64, 128, 256, 512]] + attr["layers_per_block"] = [1, 2, 3] + attr["latent_channels"] = [4, 16, 32, 64] + return attr + + def _sort_data_in_place(self, data: List[Dict], mode: str) -> None: + """ + Sort the list of design configurations in place based on a chosen mode. + + Args: + data (List[Dict]): A list of dictionaries representing design configurations. + mode (str): The sorting criterion. Can be 'abs_param_diff', 'abs_cuda_mem_diff', or 'mse'. + """ + if mode == "abs_param_diff": + data.sort(key=lambda x: abs(x["param_diff"])) + elif mode == "abs_cuda_mem_diff": + data.sort(key=lambda x: abs(x["cuda_mem_diff"])) + elif mode == "mse": + data.sort(key=lambda x: (x["param_diff"] ** 2 + x["cuda_mem_diff"] ** 2) / 2) + else: + raise ValueError("Invalid mode. Choose from 'abs_param_diff', 'abs_cuda_mem_diff', 'mse'.") + + def _print_table(self, data, headers, col_widths): + """ + Print a formatted table of the design choices. + + Args: + data (List[Dict]): The data to print, each entry a design configuration. + headers (List[str]): Column headers. + col_widths (List[int]): Widths for each column. + """ + # Create header row + header_row = "" + for header, width in zip(headers, col_widths): + header_row += f"{header:<{width}}" + print(header_row) + print("-" * sum(col_widths)) + + # Print each data row + for item in data: + row = f"{item['param_diff']:<{col_widths[0]}}" + row += f"{item['cuda_mem_diff']:<{col_widths[1]}}" + print(row) + + def search_for_target_vae(self, parameters_budget=0, cuda_max_mem=0): + """ + Search through available VAE design choices to find one that best matches + the given parameter and memory budgets. + + Args: + parameters_budget (float, optional): The target number of parameters (in millions). + cuda_max_mem (float, optional): The target maximum GPU memory usage (in MB). + + Returns: + AutoencoderKL: The chosen VAE configuration that best matches the provided budgets. + """ + if parameters_budget <= 0 and cuda_max_mem <= 0: + raise ValueError("Please specify a valid parameter budget or cuda max memory budget") + + search_space_choices = [] + if self.input_resolution == 1024 and self.compression_ratio == 8: + search_space = self._search_space_8x1024() + search_space_choices = self._generate_all_combinations(search_space) + elif self.input_resolution == 1024 and self.compression_ratio == 16: + search_space = self._search_space_16x1024() + search_space_choices = self._generate_all_combinations(search_space) + + inp_tensor = self._generate_input() + inp_tensor = inp_tensor.to(dtype=torch.float16, device="cuda") + design_choices = [] + + for choice in search_space_choices: + parameters_budget_diff = 0 + cuda_max_mem_diff = 0 + + curt_design_json = self._assign_attributes(choice) + print("-" * 20) + print(choice) + vae = AutoencoderKL.from_config(curt_design_json) + vae = vae.to(dtype=torch.float16, device="cuda") + total_params = self._count_parameters(vae) + total_params /= 10**6 + # Reset peak memory statistics + torch.cuda.reset_peak_memory_stats() + torch.cuda.synchronize() + + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + profile_memory=True, # Enables memory profiling + record_shapes=True, # Records tensor shapes + with_stack=True, # Records stack traces + ) as prof: + # Perform forward pass + start_time = time.perf_counter() + with torch.no_grad(): + _ = vae.encode(inp_tensor).latent_dist.sample() + torch.cuda.synchronize() + end_time = time.perf_counter() + + total_execution_time_ms = (end_time - start_time) * 1000 + + # Get maximum memory allocated + max_memory_allocated = torch.cuda.max_memory_allocated() + max_memory_allocated = max_memory_allocated / (1024**2) + + parameters_budget_diff = parameters_budget - total_params + cuda_max_mem_diff = cuda_max_mem - max_memory_allocated + design_choices.append( + {"param_diff": parameters_budget_diff, "cuda_mem_diff": cuda_max_mem_diff, "design": curt_design_json} + ) + + print(f" Total params: {total_params}") + print(f" Max GPU Memory Usage: {max_memory_allocated} MB") + print(f" Total Execution Time: {total_execution_time_ms:.2f} ms") + + print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=10)) + + print("-" * 20) + sort_mode = "abs_param_diff" + if parameters_budget == 0: + sort_mode = "abs_cuda_mem_diff" + elif cuda_max_mem == 0: + sort_mode = "abs_param_diff" + else: + sort_mode = "mse" + + print("#" * 20) + self._sort_data_in_place(design_choices, sort_mode) + headers = ["param_diff (M)", "cuda_mem_diff (MB)"] + col_widths = [12, 15] + self._print_table(design_choices, headers, col_widths) + + vae = AutoencoderKL.from_config(design_choices[0]["design"]) + return vae + + @property + def input_resolution(self) -> int: + """ + Get the input resolution for the VAE. + + Returns: + int: The input resolution. + """ + return self._input_resolution + + @property + def compression_ratio(self) -> float: + """ + Get the compression ratio for the VAE. + + Returns: + float: The compression ratio. + """ + return self._compression_ratio diff --git a/nemo_vfm/diffusion/vae/blocks.py b/nemo_vfm/diffusion/vae/blocks.py new file mode 100644 index 00000000..080a9685 --- /dev/null +++ b/nemo_vfm/diffusion/vae/blocks.py @@ -0,0 +1,324 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +from einops import rearrange +from torch import Tensor, nn + + +try: + from apex.contrib.group_norm import GroupNorm + + OPT_GROUP_NORM = True +except Exception: + print("Fused optimized group norm has not been installed.") + OPT_GROUP_NORM = False + + +# pylint: disable=C0116 +def Normalize(in_channels, num_groups=32, act=""): + """Creates a group normalization layer with specified activation. + + Args: + in_channels (int): Number of channels in the input. + num_groups (int, optional): Number of groups for GroupNorm. Defaults to 32. + act (str, optional): Activation function name. Defaults to "". + + Returns: + GroupNorm: A normalization layer with optional activation. + """ + return GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True, act=act) + + +def nonlinearity(x): + """Nonlinearity function used in temporal embedding projection. + + Currently implemented as a SiLU (Swish) function. + + Args: + x (Tensor): Input tensor. + + Returns: + Tensor: Output after applying SiLU activation. + """ + return x * torch.sigmoid(x) + + +class ResnetBlock(nn.Module): + """A ResNet-style block that can optionally apply a temporal embedding and shortcut projections. + + This block consists of two convolutional layers, normalization, and optional temporal embedding. + It can adjust channel dimensions between input and output via shortcuts. + """ + + def __init__(self, in_channels, out_channels=None, conv_shortcut=False, dropout=0.0, temb_channels=0): + """ + Args: + in_channels (int): Number of input channels. + out_channels (int, optional): Number of output channels. Defaults to in_channels. + conv_shortcut (bool, optional): Whether to use a convolutional shortcut. Defaults to False. + dropout (float, optional): Dropout probability. Defaults to 0.0. + temb_channels (int, optional): Number of channels in temporal embedding. Defaults to 0. + """ + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels, act="silu") + self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels, act="silu") + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x, temb): + """Forward pass of the ResnetBlock. + + Args: + x (Tensor): Input feature map of shape (B, C, H, W). + temb (Tensor): Temporal embedding tensor of shape (B, temb_channels). + + Returns: + Tensor: Output feature map of shape (B, out_channels, H, W). + """ + h = x + h = self.norm1(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class Upsample(nn.Module): + """Upsampling block that increases spatial resolution by a factor of 2. + + Can optionally include a convolution after upsampling. + """ + + def __init__(self, in_channels, with_conv): + """ + Args: + in_channels (int): Number of input channels. + with_conv (bool): If True, apply a convolution after upsampling. + """ + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x): + """Forward pass of the Upsample block. + + Args: + x (Tensor): Input feature map (B, C, H, W). + + Returns: + Tensor: Upsampled feature map (B, C, 2H, 2W). + """ + # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16 + dtype = x.dtype + if dtype == torch.bfloat16: + x = x.to(torch.float32) + x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest") + if dtype == torch.bfloat16: + x = x.to(dtype) + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + """Downsampling block that reduces spatial resolution by a factor of 2. + + Can optionally include a convolution before downsampling. + """ + + def __init__(self, in_channels, with_conv): + """ + Args: + in_channels (int): Number of input channels. + with_conv (bool): If True, apply a convolution before downsampling. + """ + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x): + """Forward pass of the Downsample block. + + Args: + x (Tensor): Input feature map (B, C, H, W). + + Returns: + Tensor: Downsampled feature map (B, C, H/2, W/2). + """ + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode="constant", value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class AttnBlock(nn.Module): + """Self-attention block that applies scaled dot-product attention to feature maps. + + Normalizes input, computes queries, keys, and values, then applies attention and a projection. + """ + + def __init__(self, in_channels: int): + """ + Args: + in_channels (int): Number of input/output channels. + """ + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels, act="silu") + + self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1) + + def attention(self, h_: Tensor) -> Tensor: + """Compute the attention over the input feature maps. + + Args: + h_ (Tensor): Normalized input feature map (B, C, H, W). + + Returns: + Tensor: Output after applying scaled dot-product attention (B, C, H, W). + """ + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + b, c, h, w = q.shape + q = rearrange(q, "b c h w -> b 1 (h w) c").contiguous() + k = rearrange(k, "b c h w -> b 1 (h w) c").contiguous() + v = rearrange(v, "b c h w -> b 1 (h w) c").contiguous() + h_ = nn.functional.scaled_dot_product_attention(q, k, v) + + return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b) + + def forward(self, x: Tensor) -> Tensor: + """Forward pass of the AttnBlock. + + Args: + x (Tensor): Input feature map (B, C, H, W). + + Returns: + Tensor: Output feature map after self-attention (B, C, H, W). + """ + return x + self.proj_out(self.attention(x)) + + +class LinearAttention(nn.Module): + """Linear Attention block for efficient attention computations. + + Uses linear attention mechanisms to reduce complexity and memory usage. + """ + + def __init__(self, dim, heads=4, dim_head=32): + """ + Args: + dim (int): Input channel dimension. + heads (int, optional): Number of attention heads. Defaults to 4. + dim_head (int, optional): Dimension per attention head. Defaults to 32. + """ + super().__init__() + self.heads = heads + hidden_dim = dim_head * heads + self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias=False) + self.to_out = nn.Conv2d(hidden_dim, dim, 1) + + def forward(self, x): + """Forward pass of the LinearAttention block. + + Args: + x (Tensor): Input feature map (B, C, H, W). + + Returns: + Tensor: Output feature map after linear attention (B, C, H, W). + """ + b, c, h, w = x.shape + qkv = self.to_qkv(x) + q, k, v = rearrange(qkv, "b (qkv heads c) h w -> qkv b heads c (h w)", heads=self.heads, qkv=3) + k = k.softmax(dim=-1) + context = torch.einsum("bhdn,bhen->bhde", k, v) + out = torch.einsum("bhde,bhdn->bhen", context, q) + out = rearrange(out, "b heads c (h w) -> b (heads c) h w", heads=self.heads, h=h, w=w) + return self.to_out(out) + + +class LinAttnBlock(LinearAttention): + """Wrapper class to provide a linear attention block in a form compatible with other attention blocks.""" + + def __init__(self, in_channels): + """ + Args: + in_channels (int): Number of input/output channels. + """ + super().__init__(dim=in_channels, heads=1, dim_head=in_channels) + + +def make_attn(in_channels, attn_type="vanilla"): + """Factory function to create an attention block. + + Args: + in_channels (int): Number of input/output channels. + attn_type (str, optional): Type of attention block to create. Options: "vanilla", "linear", "none". + Defaults to "vanilla". + + Returns: + nn.Module: An instance of the requested attention block. + """ + assert attn_type in ["vanilla", "linear", "none"], f"attn_type {attn_type} unknown" + print(f"making attention of type '{attn_type}' with {in_channels} in_channels") + if attn_type == "vanilla": + return AttnBlock(in_channels) + elif attn_type == "none": + return nn.Identity(in_channels) + else: + return LinAttnBlock(in_channels) + + +# pylint: disable=C0116 diff --git a/nemo_vfm/diffusion/vae/contperceptual_loss.py b/nemo_vfm/diffusion/vae/contperceptual_loss.py new file mode 100644 index 00000000..5b2baa83 --- /dev/null +++ b/nemo_vfm/diffusion/vae/contperceptual_loss.py @@ -0,0 +1,198 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +import torch.nn as nn +from taming.modules.losses.vqperceptual import * # TODO: taming dependency yes/no? + + +class LPIPSWithDiscriminator(nn.Module): + """ + A perceptual loss module that combines LPIPS with an adversarial discriminator + for improved reconstruction quality in variational autoencoders. This class + calculates a combination of pixel-level, perceptual (LPIPS), KL, and adversarial + losses for training a VAE model with a discriminator. + """ + + def __init__( + self, + disc_start, + logvar_init=0.0, + kl_weight=1.0, + pixelloss_weight=1.0, + disc_num_layers=3, + disc_in_channels=3, + disc_factor=1.0, + disc_weight=1.0, + perceptual_weight=1.0, + use_actnorm=False, + disc_conditional=False, + disc_loss="hinge", + ): + """ + Initializes the LPIPSWithDiscriminator module. + + Args: + disc_start (int): Iteration at which to start discriminator updates. + logvar_init (float): Initial value for the log variance parameter. + kl_weight (float): Weight for the KL divergence term. + pixelloss_weight (float): Weight for the pixel-level reconstruction loss. + disc_num_layers (int): Number of layers in the discriminator. + disc_in_channels (int): Number of input channels for the discriminator. + disc_factor (float): Scaling factor for the discriminator loss. + disc_weight (float): Weight applied to the discriminator gradient balancing. + perceptual_weight (float): Weight for the LPIPS perceptual loss. + use_actnorm (bool): Whether to use actnorm in the discriminator. + disc_conditional (bool): Whether the discriminator is conditional on an additional input. + disc_loss (str): Type of GAN loss to use ("hinge" or "vanilla"). + """ + super().__init__() + assert disc_loss in ["hinge", "vanilla"] + self.kl_weight = kl_weight + self.pixel_weight = pixelloss_weight + self.perceptual_loss = LPIPS().eval() + self.perceptual_weight = perceptual_weight + # output log variance + self.logvar = nn.Parameter(torch.ones(1) * logvar_init) + + self.discriminator = NLayerDiscriminator( + input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=use_actnorm + ).apply(weights_init) + self.discriminator_iter_start = disc_start + self.disc_loss = hinge_d_loss if disc_loss == "hinge" else vanilla_d_loss + self.disc_factor = disc_factor + self.discriminator_weight = disc_weight + self.disc_conditional = disc_conditional + + def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None): + """ + Computes an adaptive weight that balances the reconstruction (NLL) and the + adversarial (GAN) losses. This ensures stable training by adjusting the + impact of the discriminator’s gradient on the generator. + + Args: + nll_loss (torch.Tensor): The negative log-likelihood loss. + g_loss (torch.Tensor): The generator (adversarial) loss. + last_layer (torch.nn.Parameter, optional): Last layer parameters of the model + for gradient-based calculations. If None, uses self.last_layer[0]. + + Returns: + torch.Tensor: The computed adaptive weight for balancing the discriminator. + """ + if last_layer is not None: + nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0] + g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0] + else: + nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0] + g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0] + + d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4) + d_weight = torch.clamp(d_weight, 0.0, 1e4).detach() + d_weight = d_weight * self.discriminator_weight + return d_weight + + def forward( + self, inputs, reconstructions, posteriors, optimizer_idx, global_step, last_layer=None, cond=None, weights=None + ): + """ + Forward pass for computing the combined loss. Depending on the optimizer index, + this either computes the generator loss (including pixel, perceptual, KL, and + adversarial terms) or the discriminator loss. + + Args: + inputs (torch.Tensor): Original inputs to reconstruct. + reconstructions (torch.Tensor): Reconstructed outputs from the model. + posteriors (object): Posteriors from the VAE model for KL computation. + optimizer_idx (int): Indicates which optimizer is being updated + (0 for generator, 1 for discriminator). + global_step (int): Current training iteration step. + last_layer (torch.nn.Parameter, optional): The last layer's parameters for + adaptive weight calculation. + cond (torch.Tensor, optional): Conditional input for the discriminator. + weights (torch.Tensor, optional): Sample-wise weighting for the losses. + + Returns: + (torch.Tensor, dict): A tuple of (loss, log_dict) where loss is the computed loss + for the current optimizer and log_dict is a dictionary of intermediate values + for logging and debugging. + """ + rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous()) + if self.perceptual_weight > 0: + p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous()) + rec_loss = rec_loss + self.perceptual_weight * p_loss + + nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar + weighted_nll_loss = nll_loss + if weights is not None: + weighted_nll_loss = weights * nll_loss + weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0] + nll_loss = torch.sum(nll_loss) / nll_loss.shape[0] + kl_loss = posteriors.kl() + kl_loss = torch.sum(kl_loss) / kl_loss.shape[0] + + # now the GAN part + if optimizer_idx == 0: + # generator update + if cond is None: + assert not self.disc_conditional + logits_fake = self.discriminator(reconstructions.contiguous()) + else: + assert self.disc_conditional + logits_fake = self.discriminator(torch.cat((reconstructions.contiguous(), cond), dim=1)) + g_loss = -torch.mean(logits_fake) + + if self.disc_factor > 0.0: + try: + d_weight = self.calculate_adaptive_weight(nll_loss, g_loss, last_layer=last_layer) + except RuntimeError: + assert not self.training + d_weight = torch.tensor(0.0) + else: + d_weight = torch.tensor(0.0) + + disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start) + loss = weighted_nll_loss + self.kl_weight * kl_loss + d_weight * disc_factor * g_loss + + log = { + "total_loss": loss.clone().detach().mean(), + "logvar": self.logvar.detach().item(), + "kl_loss": kl_loss.detach().mean(), + "nll_loss": nll_loss.detach().mean(), + "rec_loss": rec_loss.detach().mean(), + "d_weight": d_weight.detach(), + "disc_factor": torch.tensor(disc_factor), + "g_loss": g_loss.detach().mean(), + } + return loss, log + + if optimizer_idx == 1: + # discriminator update + if cond is None: + logits_real = self.discriminator(inputs.contiguous().detach()) + logits_fake = self.discriminator(reconstructions.contiguous().detach()) + else: + logits_real = self.discriminator(torch.cat((inputs.contiguous().detach(), cond), dim=1)) + logits_fake = self.discriminator(torch.cat((reconstructions.contiguous().detach(), cond), dim=1)) + + disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start) + d_loss = disc_factor * self.disc_loss(logits_real, logits_fake) + + log = { + "disc_loss": d_loss.clone().detach().mean(), + "logits_real": logits_real.detach().mean(), + "logits_fake": logits_fake.detach().mean(), + } + return d_loss, log diff --git a/nemo_vfm/diffusion/vae/diffusers_vae.py b/nemo_vfm/diffusion/vae/diffusers_vae.py new file mode 100644 index 00000000..04b34446 --- /dev/null +++ b/nemo_vfm/diffusion/vae/diffusers_vae.py @@ -0,0 +1,36 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +from diffusers import AutoencoderKL +from einops import rearrange + + +class AutoencoderKLVAE(torch.nn.Module): + def __init__(self, path): + super().__init__() + self.vae = AutoencoderKL.from_pretrained(path, torch_dtype=torch.bfloat16) + + @torch.no_grad() + def decode(self, x): + B, C, T, H, W = x.shape + if T == 1: + x = rearrange(x, "b c t h w -> (b t) c h w") + x = x / self.vae.config.scaling_factor + out = self.vae.decode(x, return_dict=False)[0] + if T == 1: + return rearrange(out, "(b t) c h w -> b c t h w", t=1) + return out diff --git a/nemo_vfm/diffusion/vae/pretrained_vae.py b/nemo_vfm/diffusion/vae/pretrained_vae.py new file mode 100644 index 00000000..572e6a82 --- /dev/null +++ b/nemo_vfm/diffusion/vae/pretrained_vae.py @@ -0,0 +1,232 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# +# This codebase constitutes NVIDIA proprietary technology and is strictly +# confidential. Any unauthorized reproduction, distribution, or disclosure +# of this code, in whole or in part, outside NVIDIA is strictly prohibited +# without prior written consent. +# +# For inquiries regarding the use of this code in other NVIDIA proprietary +# projects, please contact the Deep Imagination Research Team at +# dir@exchange.nvidia.com. +# ----------------------------------------------------------------------------- + +# pylint: disable=C0115,C0116,C0301 + +import os +from abc import ABC, abstractmethod + +import torch + + +class BaseVAE(torch.nn.Module, ABC): + """ + Abstract base class for a Variational Autoencoder (VAE). + + All subclasses should implement the methods to define the behavior for encoding + and decoding, along with specifying the latent channel size. + """ + + def __init__(self, channel: int = 3, name: str = "vae"): + super().__init__() + self.channel = channel + self.name = name + + @property + def latent_ch(self) -> int: + """ + Returns the number of latent channels in the VAE. + """ + return self.channel + + @abstractmethod + def encode(self, state: torch.Tensor) -> torch.Tensor: + """ + Encodes the input tensor into a latent representation. + + Args: + - state (torch.Tensor): The input tensor to encode. + + Returns: + - torch.Tensor: The encoded latent tensor. + """ + pass + + @abstractmethod + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decodes the latent representation back to the original space. + + Args: + - latent (torch.Tensor): The latent tensor to decode. + + Returns: + - torch.Tensor: The decoded tensor. + """ + pass + + @property + def spatial_compression_factor(self) -> int: + """ + Returns the spatial reduction factor for the VAE. + """ + raise NotImplementedError("The spatial_compression_factor property must be implemented in the derived class.") + + +class BasePretrainedImageVAE(BaseVAE): + """ + A base class for pretrained Variational Autoencoder (VAE) that loads mean and standard deviation values + from a remote store, handles data type conversions, and normalization + using provided mean and standard deviation values for latent space representation. + Derived classes should load pre-trained encoder and decoder components from a remote store + + Attributes: + latent_mean (Tensor): The mean used for normalizing the latent representation. + latent_std (Tensor): The standard deviation used for normalizing the latent representation. + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + mean_std_fp (str): File path to the pickle file containing mean and std of the latent space. + latent_ch (int, optional): Number of latent channels (default is 16). + is_image (bool, optional): Flag to indicate whether the output is an image (default is True). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + """ + + def __init__( + self, + name: str, + mean_std_fp: str, + latent_ch: int = 16, + is_image: bool = True, + is_bf16: bool = True, + ) -> None: + super().__init__(latent_ch, name) + dtype = torch.bfloat16 if is_bf16 else torch.float32 + self.dtype = dtype + self.is_image = is_image + self.mean_std_fp = mean_std_fp + self.name = name + + self.mean_std_fp = mean_std_fp + + def register_mean_std(self, vae_dir: str) -> None: + extention = self.mean_std_fp.split(".")[-1] + latent_mean, latent_std = torch.load(os.path.join(vae_dir, f"{self.name}_mean_std.{extention}")) + + target_shape = [1, self.latent_ch, 1, 1] if self.is_image else [1, self.latent_ch, 1, 1, 1] + + self.register_buffer( + "latent_mean", + latent_mean.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + self.register_buffer( + "latent_std", + latent_std.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + """ + Encode the input state to latent space; also handle the dtype conversion, mean and std scaling + """ + in_dtype = state.dtype + latent_mean = self.latent_mean.to(in_dtype) + latent_std = self.latent_std.to(in_dtype) + encoded_state = self.encoder(state.to(self.dtype)) + if isinstance(encoded_state, torch.Tensor): + pass + elif isinstance(encoded_state, tuple): + assert isinstance(encoded_state[0], torch.Tensor) + encoded_state = encoded_state[0] + else: + raise ValueError("Invalid type of encoded state") + return (encoded_state.to(in_dtype) - latent_mean) / latent_std + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decode the input latent to state; also handle the dtype conversion, mean and std scaling + """ + in_dtype = latent.dtype + latent = latent * self.latent_std.to(in_dtype) + self.latent_mean.to(in_dtype) + return self.decoder(latent.to(self.dtype)).to(in_dtype) + + def reset_dtype(self, *args, **kwargs): + """ + Resets the data type of the encoder and decoder to the model's default data type. + + Args: + *args, **kwargs: Unused, present to allow flexibility in method calls. + """ + del args, kwargs + self.decoder.to(self.dtype) + self.encoder.to(self.dtype) + + +class JITVAE(BasePretrainedImageVAE): + """ + A JIT compiled Variational Autoencoder (VAE) that loads pre-trained encoder + and decoder components from a remote store, handles data type conversions, and normalization + using provided mean and standard deviation values for latent space representation. + + Attributes: + encoder (Module): The JIT compiled encoder loaded from storage. + decoder (Module): The JIT compiled decoder loaded from storage. + latent_mean (Tensor): The mean used for normalizing the latent representation. + latent_std (Tensor): The standard deviation used for normalizing the latent representation. + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + enc_fp (str): File path to the encoder's JIT file on the remote store. + dec_fp (str): File path to the decoder's JIT file on the remote store. + name (str): Name of the model, used for differentiating cache file paths. + mean_std_fp (str): File path to the pickle file containing mean and std of the latent space. + latent_ch (int, optional): Number of latent channels (default is 16). + is_image (bool, optional): Flag to indicate whether the output is an image (default is True). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + """ + + def __init__( + self, + enc_fp: str, + dec_fp: str, + name: str, + mean_std_fp: str, + latent_ch: int = 16, + is_image: bool = True, + is_bf16: bool = True, + ): + super().__init__(name, mean_std_fp, latent_ch, is_image, is_bf16) + self.load_encoder(enc_fp) + self.load_decoder(dec_fp) + + def load_encoder(self, vae_dir: str) -> None: + """ + Load the encoder from the remote store. + + Args: + - enc_fp (str): File path to the encoder's JIT file on the remote store. + """ + self.encoder = torch.load(vae_dir) + + self.encoder.eval() + for param in self.encoder.parameters(): + param.requires_grad = False + self.encoder.to(self.dtype) + + def load_decoder(self, vae_dir: str) -> None: + """ + Load the decoder from the remote store. + + Args: + - dec_fp (str): File path to the decoder's JIT file on the remote store. + """ + self.decoder = torch.load(vae_dir) + + self.decoder.eval() + for param in self.decoder.parameters(): + param.requires_grad = False + self.decoder.to(self.dtype) diff --git a/nemo_vfm/diffusion/vae/readme.rst b/nemo_vfm/diffusion/vae/readme.rst new file mode 100644 index 00000000..ac0f2b2f --- /dev/null +++ b/nemo_vfm/diffusion/vae/readme.rst @@ -0,0 +1,131 @@ +============================ +Pretraining Variational AutoEncoder +============================ + +Variational Autoencoder (VAE) is a data compression technique that compresses high-resolution images into a lower-dimensional latent space, capturing essential features while reducing dimensionality. This process allows for efficient storage and processing of image data. VAE has been integral to training Stable Diffusion (SD) models, significantly reducing computational requirements. For instance, SDLX utilizes a VAE that reduces image dimensions by 8x, greatly optimizing the training and inference processes. In this repository, we provide training codes to pretrain the VAE from scratch, enabling users to achieve higher compression ratios in the spatial dimension, such as 16x or 32x. + +Installation +============ + +Please pull the latest NeMo docker to get started, see details about NeMo docker `here `_. + +Validation +======== +We also provide a validation code for you to quickly evaluate our pretrained 16x VAE model on a 50K dataset. Once you start the docker, run the following script to start the testing. + +.. code-block:: bash + + torchrun --nproc-per-node 8 nemo/collections/diffusion/vae/validate_vae.py --yes data.path=path/to/validation/data log.log_dir=/path/to/checkpoint + +Configure the following variables: + + +1. ``data.path``: Set this to the directory containing your test data (e.g., `.jpg` or `.png` files). The original and VAE-reconstructed images will be logged side by side in Weights & Biases (wandb). + +2. ``log.log_dir``: Set this to the directory containing the pretrained checkpoint. You can find our pretrained checkpoint at ``TODO by ethan`` + +Here are some sample images generated from our pretrained VAE. + +``Left``: Original Image, ``Right``: 16x VAE Reconstructed Image + +.. list-table:: + :align: center + + * - .. image:: https://github.com/user-attachments/assets/08122f5b-2e65-4d65-87d7-eceae9d158fb + :width: 1400 + :align: center + - .. image:: https://github.com/user-attachments/assets/6e805a0d-8783-4d24-a65b-d96a6ba1555d + :width: 1400 + :align: center + - .. image:: https://github.com/user-attachments/assets/aab1ef33-35da-444d-90ee-ba3ad58a6b2d + :width: 1400 + :align: center + +Data Preparation +======== + +1. we expect data to be in the form of WebDataset tar files. If you have a folder of images, you can use `tar` to convert them into WebDataset tar files: + + .. code-block:: bash + + 000000.tar + ├── 1.jpg + ├── 2.jpg + 000001.tar + ├── 3.jpg + ├── 4.jpg + +2. next we need to index the webdataset with `energon `_. navigate to the dataset directory and run the following command: + + .. code-block:: bash + + energon prepare . --num-workers 8 --shuffle-tars + +3. then select dataset type `ImageWebdataset` and specify the type `jpg`. Below is an example of the interactive setup: + + .. code-block:: bash + + Found 2925 tar files in total. The first and last ones are: + - 000000.tar + - 002924.tar + If you want to exclude some of them, cancel with ctrl+c and specify an exclude filter in the command line. + Please enter a desired train/val/test split like "0.5, 0.2, 0.3" or "8,1,1": 99,1,0 + Indexing shards [####################################] 2925/2925 + Sample 0, keys: + - jpg + Sample 1, keys: + - jpg + Found the following part types in the dataset: jpg + Do you want to create a dataset.yaml interactively? [Y/n]: + The following dataset classes are available: + 0. CaptioningWebdataset + 1. CrudeWebdataset + 2. ImageClassificationWebdataset + 3. ImageWebdataset + 4. InterleavedWebdataset + 5. MultiChoiceVQAWebdataset + 6. OCRWebdataset + 7. SimilarityInterleavedWebdataset + 8. TextWebdataset + 9. VQAOCRWebdataset + 10. VQAWebdataset + 11. VidQAWebdataset + Please enter a number to choose a class: 3 + The dataset you selected uses the following sample type: + + @dataclass + class ImageSample(Sample): + """Sample type for an image, e.g. for image reconstruction.""" + + #: The input image tensor in the shape (C, H, W) + image: torch.Tensor + + Do you want to set a simple field_map[Y] (or write your own sample_loader [n])? [Y/n]: + + For each field, please specify the corresponding name in the WebDataset. + Available types in WebDataset: jpg + Leave empty for skipping optional field + You may also access json fields e.g. by setting the field to: json[field][field] + You may also specify alternative fields e.g. by setting to: jpg,png + Please enter the field_map for ImageWebdataset: + Please enter a webdataset field name for 'image' (): + That type doesn't exist in the WebDataset. Please try again. + Please enter a webdataset field name for 'image' (): jpg + Done + +4. finally, you can use the indexed dataset to train the VAE model. specify `data.path=/path/to/dataset` in the training script `train_vae.py`. + +Training +======== + +We provide a sample training script for launching multi-node training. Simply configure ``data.path`` to point to your prepared dataset to get started. + +.. code-block:: bash + + bash nemo/collections/diffusion/vae/train_vae.sh \ + data.path=xxx + + + + + diff --git a/nemo_vfm/diffusion/vae/test_autovae.py b/nemo_vfm/diffusion/vae/test_autovae.py new file mode 100644 index 00000000..022a218f --- /dev/null +++ b/nemo_vfm/diffusion/vae/test_autovae.py @@ -0,0 +1,160 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import unittest + +import torch +from autovae import VAEGenerator + + +class TestVAEGenerator(unittest.TestCase): + """Unit tests for the VAEGenerator class.""" + + def setUp(self): + # Common setup for tests + self.input_resolution = 1024 + self.compression_ratio = 8 + self.generator = VAEGenerator(input_resolution=self.input_resolution, compression_ratio=self.compression_ratio) + + def test_initialization_valid(self): + """Test that valid initialization parameters set the correct properties.""" + generator = VAEGenerator(input_resolution=1024, compression_ratio=8) + self.assertEqual(generator.input_resolution, 1024) + self.assertEqual(generator.compression_ratio, 8) + + generator = VAEGenerator(input_resolution=2048, compression_ratio=16) + self.assertEqual(generator.input_resolution, 2048) + self.assertEqual(generator.compression_ratio, 16) + + def test_initialization_invalid(self): + """Test that invalid initialization parameters raise an error.""" + with self.assertRaises(NotImplementedError): + VAEGenerator(input_resolution=4096, compression_ratio=16) + + def test_generate_input(self): + """Test that _generate_input produces a tensor with the correct shape and device.""" + input_tensor = self.generator._generate_input() + expected_shape = (1, 3, self.input_resolution, self.input_resolution) + self.assertEqual(input_tensor.shape, expected_shape) + self.assertEqual(input_tensor.dtype, torch.float16) + self.assertEqual(input_tensor.device.type, "cuda") + + def test_count_parameters(self): + """Test that _count_parameters correctly counts model parameters.""" + model = torch.nn.Sequential(torch.nn.Linear(10, 20), torch.nn.ReLU(), torch.nn.Linear(20, 5)) + expected_param_count = sum(p.numel() for p in model.parameters() if p.requires_grad) + param_count = self.generator._count_parameters(model) + self.assertEqual(param_count, expected_param_count) + + def test_load_base_json_skeleton(self): + """Test that _load_base_json_skeleton returns the correct skeleton.""" + skeleton = self.generator._load_base_json_skeleton() + expected_keys = { + "_class_name", + "_diffusers_version", + "_name_or_path", + "act_fn", + "block_out_channels", + "down_block_types", + "force_upcast", + "in_channels", + "latent_channels", + "layers_per_block", + "norm_num_groups", + "out_channels", + "sample_size", + "scaling_factor", + "up_block_types", + } + self.assertEqual(set(skeleton.keys()), expected_keys) + + def test_generate_all_combinations(self): + """Test that _generate_all_combinations generates all possible combinations.""" + attr = {"layers_per_block": [1, 2], "latent_channels": [4, 8]} + combinations = self.generator._generate_all_combinations(attr) + expected_combinations = [ + {"layers_per_block": 1, "latent_channels": 4}, + {"layers_per_block": 1, "latent_channels": 8}, + {"layers_per_block": 2, "latent_channels": 4}, + {"layers_per_block": 2, "latent_channels": 8}, + ] + self.assertEqual(len(combinations), len(expected_combinations)) + for combo in expected_combinations: + self.assertIn(combo, combinations) + + def test_assign_attributes(self): + """Test that _assign_attributes correctly assigns attributes to the skeleton.""" + choice = { + "down_block_types": ["DownEncoderBlock2D"] * 4, + "up_block_types": ["UpDecoderBlock2D"] * 4, + "block_out_channels": [64, 128, 256, 512], + "layers_per_block": 2, + "latent_channels": 16, + } + skeleton = self.generator._assign_attributes(choice) + self.assertEqual(skeleton["down_block_types"], choice["down_block_types"]) + self.assertEqual(skeleton["up_block_types"], choice["up_block_types"]) + self.assertEqual(skeleton["block_out_channels"], choice["block_out_channels"]) + self.assertEqual(skeleton["layers_per_block"], choice["layers_per_block"]) + self.assertEqual(skeleton["latent_channels"], choice["latent_channels"]) + + def test_search_space_16x1024(self): + """Test that _search_space_16x1024 returns the correct search space.""" + search_space = self.generator._search_space_16x1024() + expected_keys = { + "down_block_types", + "up_block_types", + "block_out_channels", + "layers_per_block", + "latent_channels", + } + self.assertEqual(set(search_space.keys()), expected_keys) + self.assertTrue(all(isinstance(v, list) for v in search_space.values())) + + def test_sort_data_in_place(self): + """Test that _sort_data_in_place correctly sorts data based on the specified mode.""" + data = [ + {"param_diff": 10, "cuda_mem_diff": 100}, + {"param_diff": 5, "cuda_mem_diff": 50}, + {"param_diff": -3, "cuda_mem_diff": 30}, + {"param_diff": 7, "cuda_mem_diff": 70}, + ] + # Test sorting by absolute parameter difference + self.generator._sort_data_in_place(data, mode="abs_param_diff") + expected_order_param = [-3, 5, 7, 10] + actual_order_param = [item["param_diff"] for item in data] + self.assertEqual(actual_order_param, expected_order_param) + + # Test sorting by absolute CUDA memory difference + self.generator._sort_data_in_place(data, mode="abs_cuda_mem_diff") + expected_order_mem = [30, 50, 70, 100] + actual_order_mem = [item["cuda_mem_diff"] for item in data] + self.assertEqual(actual_order_mem, expected_order_mem) + + # Test sorting by mean squared error (MSE) + self.generator._sort_data_in_place(data, mode="mse") + expected_order_mse = [-3, 5, 7, 10] # Computed based on MSE values + actual_order_mse = [item["param_diff"] for item in data] + self.assertEqual(actual_order_mse, expected_order_mse) + + def test_search_for_target_vae_invalid(self): + """Test that search_for_target_vae raises an error when no budget is specified.""" + with self.assertRaises(ValueError): + self.generator.search_for_target_vae(parameters_budget=0, cuda_max_mem=0) + + +if __name__ == "__main__": + unittest.main() diff --git a/nemo_vfm/diffusion/vae/train_vae.py b/nemo_vfm/diffusion/vae/train_vae.py new file mode 100644 index 00000000..37aa7562 --- /dev/null +++ b/nemo_vfm/diffusion/vae/train_vae.py @@ -0,0 +1,366 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + + +from typing import Any, Callable, Dict, Optional, Sequence, Tuple + +import nemo_run as run +import torch +import torch.distributed +import torch.utils.checkpoint +import torchvision +import wandb +from autovae import VAEGenerator +from contperceptual_loss import LPIPSWithDiscriminator +from diffusers import AutoencoderKL +from megatron.core import parallel_state +from megatron.core.transformer.enums import ModelType +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.energon import DefaultTaskEncoder, ImageSample +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.data.diffusion_energon_datamodule import DiffusionDataModule +from nemo.collections.diffusion.train import pretrain +from nemo.collections.llm.gpt.model.base import GPTModel +from nemo.lightning.io.mixin import IOMixin +from nemo.lightning.megatron_parallel import DataT, MegatronLossReduction, ReductionT +from nemo.lightning.pytorch.optim import OptimizerModule +from torch import Tensor, nn + + +class AvgLossReduction(MegatronLossReduction): + """Performs average loss reduction across micro-batches.""" + + def forward(self, batch: DataT, forward_out: Tensor) -> Tuple[Tensor, ReductionT]: + """ + Forward pass for loss reduction. + + Args: + batch: The batch of data. + forward_out: The output tensor from forward computation. + + Returns: + A tuple of (loss, reduction dictionary). + """ + loss = forward_out.mean() + return loss, {"avg": loss} + + def reduce(self, losses_reduced_per_micro_batch: Sequence[ReductionT]) -> Tensor: + """ + Reduce losses across multiple micro-batches by averaging them. + + Args: + losses_reduced_per_micro_batch: A sequence of loss dictionaries. + + Returns: + The averaged loss tensor. + """ + losses = torch.stack([loss["avg"] for loss in losses_reduced_per_micro_batch]) + return losses.mean() + + +class VAE(MegatronModule): + """Variational Autoencoder (VAE) module.""" + + def __init__(self, config, pretrained_model_name_or_path, search_vae=False): + """ + Initialize the VAE model. + + Args: + config: Transformer configuration. + pretrained_model_name_or_path: Path or name of the pretrained model. + search_vae: Flag to indicate whether to search for a target VAE using AutoVAE. + """ + super().__init__(config) + if search_vae: + # Get VAE automatically from AutoVAE + self.vae = VAEGenerator(input_resolution=1024, compression_ratio=16) + # Below line is commented out due to an undefined 'generator' variable in original code snippet. + # self.vae = generator.search_for_target_vae(parameters_budget=895.178707, cuda_max_mem=0) + else: + self.vae = AutoencoderKL.from_config(pretrained_model_name_or_path, weight_dtype=torch.bfloat16) + + sdxl_vae = AutoencoderKL.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", subfolder="vae", weight_dtype=torch.bfloat16 + ) + sd_dict = sdxl_vae.state_dict() + vae_dict = self.vae.state_dict() + pre_dict = {k: v for k, v in sd_dict.items() if (k in vae_dict) and (vae_dict[k].numel() == v.numel())} + self.vae.load_state_dict(pre_dict, strict=False) + del sdxl_vae + + self.vae_loss = LPIPSWithDiscriminator( + disc_start=50001, + logvar_init=0.0, + kl_weight=0.000001, + pixelloss_weight=1.0, + disc_num_layers=3, + disc_in_channels=3, + disc_factor=1.0, + disc_weight=0.5, + perceptual_weight=1.0, + use_actnorm=False, + disc_conditional=False, + disc_loss="hinge", + ) + + def forward(self, target, global_step): + """ + Forward pass through the VAE. + + Args: + target: Target images. + global_step: Current global step. + + Returns: + A tuple (aeloss, log_dict_ae, pred) containing the loss, log dictionary, and predictions. + """ + posterior = self.vae.encode(target).latent_dist + z = posterior.sample() + pred = self.vae.decode(z).sample + aeloss, log_dict_ae = self.vae_loss( + inputs=target, + reconstructions=pred, + posteriors=posterior, + optimizer_idx=0, + global_step=global_step, + last_layer=self.vae.decoder.conv_out.weight, + ) + return aeloss, log_dict_ae, pred + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """ + Set input tensor. + + Args: + input_tensor: The input tensor to the model. + """ + pass + + +class VAEModel(GPTModel): + """A GPTModel wrapper for the VAE.""" + + def __init__( + self, + pretrained_model_name_or_path: str, + optim: Optional[OptimizerModule] = None, + model_transform: Optional[Callable[[nn.Module], nn.Module]] = None, + ): + """ + Initialize the VAEModel. + + Args: + pretrained_model_name_or_path: Path or name of the pretrained model. + optim: Optional optimizer module. + model_transform: Optional function to transform the model. + """ + self.pretrained_model_name_or_path = pretrained_model_name_or_path + config = TransformerConfig(num_layers=1, hidden_size=1, num_attention_heads=1) + self.model_type = ModelType.encoder_or_decoder + super().__init__(config, optim=optim, model_transform=model_transform) + + def configure_model(self) -> None: + """Configure the model by initializing the module.""" + if not hasattr(self, "module"): + self.module = VAE(self.config, self.pretrained_model_name_or_path) + + def data_step(self, dataloader_iter) -> Dict[str, Any]: + """ + Perform a single data step to fetch a batch from the iterator. + + Args: + dataloader_iter: The dataloader iterator. + + Returns: + A dictionary with 'pixel_values' ready for the model. + """ + batch = next(dataloader_iter)[0] + return {"pixel_values": batch.image.to(device="cuda", dtype=torch.bfloat16, non_blocking=True)} + + def forward(self, *args, **kwargs): + """ + Forward pass through the underlying module. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + The result of forward pass of self.module. + """ + return self.module(*args, **kwargs) + + def training_step(self, batch, batch_idx=None) -> torch.Tensor: + """ + Perform a single training step. + + Args: + batch: The input batch. + batch_idx: Batch index. + + Returns: + The loss tensor. + """ + loss, log_dict_ae, pred = self(batch["pixel_values"], self.global_step) + + if torch.distributed.get_rank() == 0: + self.log_dict(log_dict_ae) + + return loss + + def validation_step(self, batch, batch_idx=None) -> torch.Tensor: + """ + Perform a single validation step. + + Args: + batch: The input batch. + batch_idx: Batch index. + + Returns: + The loss tensor. + """ + loss, log_dict_ae, pred = self(batch["pixel_values"], self.global_step) + + image = torch.cat([batch["pixel_values"].cpu(), pred.cpu()], axis=0) + image = (image + 0.5).clamp(0, 1) + + # wandb is on the last rank for megatron, first rank for nemo + wandb_rank = 0 + + if parallel_state.get_data_parallel_src_rank() == wandb_rank: + if torch.distributed.get_rank() == wandb_rank: + gather_list = [None for _ in range(parallel_state.get_data_parallel_world_size())] + else: + gather_list = None + torch.distributed.gather_object( + image, gather_list, wandb_rank, group=parallel_state.get_data_parallel_group() + ) + if gather_list is not None: + self.log_dict(log_dict_ae) + wandb.log( + { + "Original (left), Reconstruction (right)": [ + wandb.Image(torchvision.utils.make_grid(image)) for _, image in enumerate(gather_list) + ] + }, + ) + + return loss + + @property + def training_loss_reduction(self) -> AvgLossReduction: + """Returns the loss reduction method for training.""" + if not self._training_loss_reduction: + self._training_loss_reduction = AvgLossReduction() + return self._training_loss_reduction + + @property + def validation_loss_reduction(self) -> AvgLossReduction: + """Returns the loss reduction method for validation.""" + if not self._validation_loss_reduction: + self._validation_loss_reduction = AvgLossReduction() + return self._validation_loss_reduction + + def on_validation_model_zero_grad(self) -> None: + """ + Hook to handle zero grad on validation model step. + Used here to skip first validation on resume. + """ + super().on_validation_model_zero_grad() + if self.trainer.ckpt_path is not None and getattr(self, "_restarting_skip_val_flag", True): + self.trainer.sanity_checking = True + self._restarting_skip_val_flag = False + + +def crop_image(img, divisor=16): + """ + Crop the image so that both dimensions are divisible by the given divisor. + + Args: + img: Image tensor. + divisor: The divisor to use for cropping. + + Returns: + The cropped image tensor. + """ + h, w = img.shape[-2], img.shape[-1] + + delta_h = h % divisor + delta_w = w % divisor + + delta_h_top = delta_h // 2 + delta_h_bottom = delta_h - delta_h_top + + delta_w_left = delta_w // 2 + delta_w_right = delta_w - delta_w_left + + img_cropped = img[..., delta_h_top : h - delta_h_bottom, delta_w_left : w - delta_w_right] + + return img_cropped + + +class ImageTaskEncoder(DefaultTaskEncoder, IOMixin): + """Image task encoder that crops and normalizes the image.""" + + def encode_sample(self, sample: ImageSample) -> ImageSample: + """ + Encode a single image sample by cropping and shifting its values. + + Args: + sample: An image sample. + + Returns: + The transformed image sample. + """ + sample = super().encode_sample(sample) + sample.image = crop_image(sample.image, 16) + sample.image -= 0.5 + return sample + + +@run.cli.factory(target=llm.train) +def train_vae() -> run.Partial: + """ + Training factory function for VAE. + + Returns: + A run.Partial recipe for training. + """ + recipe = pretrain() + recipe.model = run.Config( + VAEModel, + pretrained_model_name_or_path="nemo/collections/diffusion/vae/vae16x/config.json", + ) + recipe.data = run.Config( + DiffusionDataModule, + task_encoder=run.Config(ImageTaskEncoder), + global_batch_size=24, + num_workers=10, + ) + recipe.optim.lr_scheduler = run.Config(nl.lr_scheduler.WarmupHoldPolicyScheduler, warmup_steps=100, hold_steps=1e9) + recipe.optim.config.lr = 5e-6 + recipe.optim.config.weight_decay = 1e-2 + recipe.log.log_dir = "nemo_experiments/train_vae" + recipe.trainer.val_check_interval = 1000 + recipe.trainer.callbacks[0].every_n_train_steps = 1000 + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=train_vae) diff --git a/nemo_vfm/diffusion/vae/train_vae.sh b/nemo_vfm/diffusion/vae/train_vae.sh new file mode 100644 index 00000000..3f5a46ab --- /dev/null +++ b/nemo_vfm/diffusion/vae/train_vae.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#SBATCH -p batch -A coreai_dlalgo_llm -t 4:00:00 --nodes=16 --exclusive --mem=0 --overcommit --gpus-per-node 8 --ntasks-per-node=8 --dependency=singleton + +export WANDB_RESUME=allow +export WANDB_NAME=train_vae + +DIR=`pwd` + +srun --signal=TERM@300 -l --container-image ${IMAGE} --container-mounts "/lustre:/lustre/,/home:/home" --no-container-mount-home --mpi=pmix bash -c "cd ${DIR} ; python -u nemo/collections/diffusion/vae/train_vae.py --yes $*" diff --git a/nemo_vfm/diffusion/vae/vae16x/config.json b/nemo_vfm/diffusion/vae/vae16x/config.json new file mode 100644 index 00000000..9b363564 --- /dev/null +++ b/nemo_vfm/diffusion/vae/vae16x/config.json @@ -0,0 +1,35 @@ +{ + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.20.0.dev0", + "_name_or_path": "../sdxl-vae/", + "act_fn": "silu", + "block_out_channels": [ + 128, + 256, + 512, + 1024, + 2048 + ], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D" + ], + "force_upcast": false, + "in_channels": 3, + "latent_channels": 16, + "layers_per_block": 2, + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 1024, + "scaling_factor": 0.13025, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D" + ] + } diff --git a/nemo_vfm/diffusion/vae/validate_vae.py b/nemo_vfm/diffusion/vae/validate_vae.py new file mode 100644 index 00000000..dd143d9e --- /dev/null +++ b/nemo_vfm/diffusion/vae/validate_vae.py @@ -0,0 +1,49 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nemo_run as run +from nemo.collections import llm +from nemo.collections.diffusion.vae.train_vae import train_vae + + +@run.cli.factory(target=llm.validate) +def validate_vae() -> run.Partial: + """ + Create a partial function for validating a VAE (Variational Autoencoder) model. + + This function uses the training recipe defined in `train_vae()` to set up + the model, data, trainer, logging, and optimization configurations for + validation. It returns a Partial object that can be used by the NeMo run CLI + to execute the validation procedure on the provided model and data. + + Returns: + run.Partial: A partial object configured with llm.validate target + and all necessary arguments extracted from the VAE training recipe. + """ + recipe = train_vae() + return run.Partial( + llm.validate, + model=recipe.model, + data=recipe.data, + trainer=recipe.trainer, + log=recipe.log, + optim=recipe.optim, + tokenizer=None, + resume=recipe.resume, + model_transform=None, + ) + + +if __name__ == "__main__": + run.cli.main(llm.validate, default_factory=validate_vae) diff --git a/nemo_vfm/diffusion/vae/video_vae.py b/nemo_vfm/diffusion/vae/video_vae.py new file mode 100644 index 00000000..9ddbbea6 --- /dev/null +++ b/nemo_vfm/diffusion/vae/video_vae.py @@ -0,0 +1,447 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# +# This codebase constitutes NVIDIA proprietary technology and is strictly +# confidential. Any unauthorized reproduction, distribution, or disclosure +# of this code, in whole or in part, outside NVIDIA is strictly prohibited +# without prior written consent. +# +# For inquiries regarding the use of this code in other NVIDIA proprietary +# projects, please contact the Deep Imagination Research Team at +# dir@exchange.nvidia.com. +# ----------------------------------------------------------------------------- + +# pylint: disable=C0115,C0116,C0301 + +import os +from abc import ABC, abstractmethod + +import torch +from einops import rearrange +from huggingface_hub import snapshot_download +from nemo.collections.diffusion.vae.pretrained_vae import JITVAE, BaseVAE +from torch.nn.modules import Module + + +class VideoTokenizerInterface(ABC): + @abstractmethod + def encode(self, state: torch.Tensor) -> torch.Tensor: + pass + + @abstractmethod + def decode(self, latent: torch.Tensor) -> torch.Tensor: + pass + + @abstractmethod + def get_latent_num_frames(self, num_pixel_frames: int) -> int: + pass + + @abstractmethod + def get_pixel_num_frames(self, num_latent_frames: int) -> int: + pass + + @property + @abstractmethod + def spatial_compression_factor(self): + pass + + @property + @abstractmethod + def temporal_compression_factor(self): + pass + + @property + @abstractmethod + def spatial_resolution(self): + pass + + @property + @abstractmethod + def pixel_chunk_duration(self): + pass + + @property + @abstractmethod + def latent_chunk_duration(self): + pass + + +class BasePretrainedVideoTokenizer(ABC): + """ + Base class for a pretrained video tokenizer that handles chunking of video data for efficient processing. + + Args: + pixel_chunk_duration (int): The duration (in number of frames) of each chunk of video data at the pixel level. + temporal_compress_factor (int): The factor by which the video data is temporally compressed during processing. + max_enc_batch_size (int): The maximum batch size to process in one go during encoding to avoid memory overflow. + max_dec_batch_size (int): The maximum batch size to process in one go during decoding to avoid memory overflow. + + The class introduces parameters for managing temporal chunks (`pixel_chunk_duration` and `temporal_compress_factor`) + which define how video data is subdivided and compressed during the encoding and decoding processes. The + `max_enc_batch_size` and `max_dec_batch_size` parameters allow processing in smaller batches to handle memory + constraints. + """ + + def __init__( + self, + pixel_chunk_duration: int = 17, + temporal_compress_factor: int = 8, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + ): + self._pixel_chunk_duration = pixel_chunk_duration + self._temporal_compress_factor = temporal_compress_factor + self.max_enc_batch_size = max_enc_batch_size + self.max_dec_batch_size = max_dec_batch_size + + def register_mean_std(self, vae_dir: str) -> None: + extension = self.mean_std_fp.split(".")[-1] + latent_mean, latent_std = torch.load(os.path.join(vae_dir, f"mean_std.{extension}")) + + latent_mean = latent_mean.view(self.latent_ch, -1)[:, : self.latent_chunk_duration] + latent_std = latent_std.view(self.latent_ch, -1)[:, : self.latent_chunk_duration] + + target_shape = [1, self.latent_ch, self.latent_chunk_duration, 1, 1] + + self.register_buffer( + "latent_mean", + latent_mean.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + self.register_buffer( + "latent_std", + latent_std.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + + def transform_encode_state_shape(self, state: torch.Tensor) -> torch.Tensor: + """ + Rearranges the input state tensor to the required shape for encoding video data. Mainly for chunk based encoding + """ + B, C, T, H, W = state.shape + # assert ( + # T % self.pixel_chunk_duration == 0 + # ), f"Temporal dimension {T} is not divisible by chunk_length {self.pixel_chunk_duration}" + return rearrange(state, "b c (n t) h w -> (b n) c t h w", t=T) + + def transform_decode_state_shape(self, latent: torch.Tensor) -> torch.Tensor: + B, _, T, _, _ = latent.shape + # assert ( + # T % self.latent_chunk_duration == 0 + # ), f"Temporal dimension {T} is not divisible by chunk_length {self.latent_chunk_duration}" + return rearrange(latent, "b c (n t) h w -> (b n) c t h w", t=T) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + origin_T = None + if self._temporal_compress_factor == 1: + _, _, origin_T, _, _ = state.shape + state = rearrange(state, "b c t h w -> (b t) c 1 h w") + B, C, T, H, W = state.shape + state = self.transform_encode_state_shape(state) + # use max_enc_batch_size to avoid OOM + if state.shape[0] > self.max_enc_batch_size: + latent = [] + for i in range(0, state.shape[0], self.max_enc_batch_size): + latent.append(super().encode(state[i : i + self.max_enc_batch_size])) + latent = torch.cat(latent, dim=0) + else: + latent = super().encode(state) + + latent = rearrange(latent, "(b n) c t h w -> b c (n t) h w", b=B) + if self._temporal_compress_factor == 1: + latent = rearrange(latent, "(b t) c 1 h w -> b c t h w", t=origin_T) + return latent + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decodes a batch of latent representations into video frames by applying temporal chunking. Similar to encode, + it handles video data by processing smaller temporal chunks to reconstruct the original video dimensions. + + It can also decode single frame image data. + + Args: + latent (torch.Tensor): The latent space tensor containing encoded video data. + + Returns: + torch.Tensor: The decoded video tensor reconstructed from latent space. + """ + origin_T = None + if self._temporal_compress_factor == 1: + _, _, origin_T, _, _ = latent.shape + latent = rearrange(latent, "b c t h w -> (b t) c 1 h w") + B, _, T, _, _ = latent.shape + latent = self.transform_decode_state_shape(latent) + # use max_enc_batch_size to avoid OOM + if latent.shape[0] > self.max_dec_batch_size: + state = [] + for i in range(0, latent.shape[0], self.max_dec_batch_size): + state.append(super().decode(latent[i : i + self.max_dec_batch_size])) + state = torch.cat(state, dim=0) + else: + state = super().decode(latent) + assert state.shape[2] == self.pixel_chunk_duration + state = rearrange(state, "(b n) c t h w -> b c (n t) h w", b=B) + if self._temporal_compress_factor == 1: + return rearrange(state, "(b t) c 1 h w -> b c t h w", t=origin_T) + return state + + @property + def pixel_chunk_duration(self) -> int: + return self._pixel_chunk_duration + + @property + def latent_chunk_duration(self) -> int: + # return self._latent_chunk_duration + assert (self.pixel_chunk_duration - 1) % self.temporal_compression_factor == 0, ( + f"Pixel chunk duration {self.pixel_chunk_duration} is not divisible by latent chunk duration " + f"{self.latent_chunk_duration}" + ) + return (self.pixel_chunk_duration - 1) // self.temporal_compression_factor + 1 + + @property + def temporal_compression_factor(self): + return self._temporal_compress_factor + + def get_latent_num_frames(self, num_pixel_frames: int) -> int: + if num_pixel_frames == 1: + return 1 + assert num_pixel_frames % self.pixel_chunk_duration == 0, ( + f"Temporal dimension {num_pixel_frames} is not divisible by chunk_length {self.pixel_chunk_duration}" + ) + return num_pixel_frames // self.pixel_chunk_duration * self.latent_chunk_duration + + def get_pixel_num_frames(self, num_latent_frames: int) -> int: + if num_latent_frames == 1: + return 1 + assert num_latent_frames % self.latent_chunk_duration == 0, ( + f"Temporal dimension {num_latent_frames} is not divisible by chunk_length {self.latent_chunk_duration}" + ) + return num_latent_frames // self.latent_chunk_duration * self.pixel_chunk_duration + + +class VideoJITTokenizer(BasePretrainedVideoTokenizer, JITVAE, VideoTokenizerInterface): + """ + Instance of BasePretrainedVideoVAE that loads encoder and decoder from JIT scripted module file + """ + + def __init__( + self, + vae_path: str, + enc_fp: str, + dec_fp: str, + name: str, + mean_std_fp: str, + latent_ch: int = 16, + is_bf16: bool = True, + spatial_compression_factor: int = 16, + temporal_compression_factor: int = 8, + pixel_chunk_duration: int = 17, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + spatial_resolution: str = "720", + ): + super().__init__( + pixel_chunk_duration, + temporal_compression_factor, + max_enc_batch_size, + max_dec_batch_size, + ) + super(BasePretrainedVideoTokenizer, self).__init__( + enc_fp, + dec_fp, + name, + mean_std_fp, + latent_ch, + False, + is_bf16, + ) + + self._spatial_compression_factor = spatial_compression_factor + self._spatial_resolution = spatial_resolution + + self.register_mean_std(vae_path) + + @property + def spatial_compression_factor(self): + return self._spatial_compression_factor + + @property + def spatial_resolution(self) -> str: + return self._spatial_resolution + + +class JointImageVideoTokenizer(BaseVAE, VideoTokenizerInterface): + def __init__( + self, + image_vae: torch.nn.Module, + video_vae: torch.nn.Module, + name: str, + latent_ch: int = 16, + squeeze_for_image: bool = True, + ): + super().__init__(latent_ch, name) + self.image_vae = image_vae + self.video_vae = video_vae + self.squeeze_for_image = squeeze_for_image + + def encode_image(self, state: torch.Tensor) -> torch.Tensor: + if self.squeeze_for_image: + return self.image_vae.encode(state.squeeze(2)).unsqueeze(2) + return self.image_vae.encode(state) + + def decode_image(self, latent: torch.Tensor) -> torch.Tensor: + if self.squeeze_for_image: + return self.image_vae.decode(latent.squeeze(2)).unsqueeze(2) + return self.image_vae.decode(latent) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + B, C, T, H, W = state.shape + if T == 1: + return self.encode_image(state) + + return self.video_vae.encode(state) + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + B, C, T, H, W = latent.shape + if T == 1: + return self.decode_image(latent) + return self.video_vae.decode(latent) + + def reset_dtype(self, *args, **kwargs): + """ + Resets the data type of the encoder and decoder to the model's default data type. + + Args: + *args, **kwargs: Unused, present to allow flexibility in method calls. + """ + del args, kwargs + self.image_vae.reset_dtype() + self.video_vae.reset_dtype() + + def get_latent_num_frames(self, num_pixel_frames: int) -> int: + if num_pixel_frames == 1: + return 1 + return self.video_vae.get_latent_num_frames(num_pixel_frames) + + def get_pixel_num_frames(self, num_latent_frames: int) -> int: + if num_latent_frames == 1: + return 1 + return self.video_vae.get_pixel_num_frames(num_latent_frames) + + @property + def spatial_compression_factor(self): + return self.video_vae.spatial_compression_factor + + @property + def temporal_compression_factor(self): + return self.video_vae.temporal_compression_factor + + @property + def spatial_resolution(self) -> str: + return self.video_vae.spatial_resolution + + @property + def pixel_chunk_duration(self) -> int: + return self.video_vae.pixel_chunk_duration + + @property + def latent_chunk_duration(self) -> int: + return self.video_vae.latent_chunk_duration + + +class JointImageVideoSharedJITTokenizer(JointImageVideoTokenizer): + """ + First version of the ImageVideoVAE trained with Fitsum. + We have to use seperate mean and std for image and video due to non-causal nature of the model. + """ + + def __init__(self, image_vae: Module, video_vae: Module, name: str, latent_ch: int = 16): + super().__init__(image_vae, video_vae, name, latent_ch, squeeze_for_image=False) + assert isinstance(image_vae, JITVAE) + assert isinstance(video_vae, VideoJITTokenizer), ( + f"video_vae should be an instance of VideoJITVAE, got {type(video_vae)}" + ) + # a hack to make the image_vae and video_vae share the same encoder and decoder + + def load_weights(self, vae_dir: str): + self.video_vae.register_mean_std(vae_dir) + self.image_vae.register_mean_std(vae_dir) + + self.video_vae.load_decoder(vae_dir) + self.video_vae.load_encoder(vae_dir) + + self.image_vae.encoder = self.video_vae.encoder + self.image_vae.decoder = self.video_vae.decoder + + +def video_vae3_512( + vae_path: str, + enc_fp: str = None, + dec_fp: str = None, + mean_std_fp: str = None, + latent_ch: int = 16, + is_bf16: bool = True, + video_mean_std_fp=None, + image_mean_std_fp=None, + spatial_compression_factor: int = 16, + temporal_compression_factor: int = 8, + pixel_chunk_duration: int = 121, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + spatial_resolution: str = "720", +): + name = "cosmos_tokenizer" + if enc_fp is None: + enc_fp = os.path.join(vae_path, "encoder.jit") + if dec_fp is None: + dec_fp = os.path.join(vae_path, "decoder.jit") + if video_mean_std_fp is None: + video_mean_std_fp = os.path.join(vae_path, "mean_std.pt") + + video_vae = VideoJITTokenizer( + vae_path, + enc_fp, + dec_fp, + name, + video_mean_std_fp, + pixel_chunk_duration=pixel_chunk_duration, + spatial_compression_factor=8, + temporal_compression_factor=8, + ) + + image_vae = VideoJITTokenizer( + vae_path, + enc_fp, + dec_fp, + name, + video_mean_std_fp, + pixel_chunk_duration=1, + spatial_compression_factor=8, + temporal_compression_factor=8, + ) + + video_image_vae = JointImageVideoSharedJITTokenizer( + image_vae, + video_vae, + name, + ) + + return video_image_vae + + +if __name__ == "__main__": + tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + vae = video_vae3_512(vae_path=tokenizer_dir) + + image = torch.randn(1, 3, 1, 704, 1280, device="cuda", dtype=torch.bfloat16) + latent = vae.encode(image) + print(latent.shape) + + video = torch.randn(1, 3, 121, 704, 1280, device="cuda", dtype=torch.bfloat16) + latent = vae.encode(video) + print(latent.shape) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/__init__.py new file mode 100644 index 00000000..dac9a4d7 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/POST_TRAINING.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/POST_TRAINING.md new file mode 100644 index 00000000..cc7a95a3 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/POST_TRAINING.md @@ -0,0 +1,25 @@ +# Cosmos Post-training + +In the [Cosmos paper](https://research.nvidia.com/publication/2025-01_cosmos-world-foundation-model-platform-physical-ai), we discuss several post-training examples of Cosmos pre-trained World Foundation Models (WFMs) for various Physical AI tasks, including + +- General Post-Training: Fine-tune the WFM to generate a target distribution of videos based on the custom dataset. The target distribution could include a specific camera spec or a specific domain such as a factory. +- Instruction Control: Post-trains models for robotic manipulation to predict videos based on textual instructions, enabling robots to visually simulate tasks like folding clothes or picking up objects. +- Action Control: Post-trains models for robotic manipulation to predict the next visual frame based on action vectors, simulating robotic tasks like object handling or movement planning. +- Camera Control: Adds camera pose conditioning to generate 3D-consistent video simulations from single images, enabling joystick-like navigation in virtual environments. +- Multi-View Generation: Post-trains models for autonomous vehicles to generate synchronized multi-view videos from text prompts, simulating driving scenarios with multiple camera perspectives. +- Multi-View Generation with Vehicle Trajectory Control: Extends multi-view generation by incorporating trajectory inputs, enabling precise simulation of driving environments for autonomous vehicles, adhering to specified paths. +- Changing the Video Tokenizer: Post-train the WFM to adapt to a new tokenizer. e.g. from 8x8x8 to 4×8×8. + +Except for the instruction control where the WFM is post-trained on a dataset of instruction-video pairs, all other cases require minor modifications of the network architectures. Post-training tasks will be supported by NeMo Framework. In this initial release, we provide post-training scripts for the general post-training of both diffusion and autorgressive WFMs. Scripts of the other post-training tasks will be provided in a future release. + +## Post-training Support Matrix + +| Post-training Task | Diffusion WFM | Autoregressive WFM | +|---------------------|---------------|--------------------| +| General post-training | [Supported](../models/diffusion/nemo/post_training/README.md) | [Supported](../models/autoregressive/nemo/post_training/README.md) | +| Instruction control | [Supported](./diffusion/nemo/post_training/README.md) | [Supported](./diffusion/autoregressive/post_training/README.md) | +| Action control | [Supported](./diffusion/nemo/post_training/action_control/README.md) | [Supported](./autoregressive/nemo/post_training/action_control/README.md) | +| Camera control | [Supported](./diffusion/nemo/post_training/camera_control/README.md) | Coming soon | +| Multi-view generation | [Supported](./diffusion/nemo/post_training/README.md) | Coming soon | +| Multi-view generation with vehicle trajectory control | [Supported](./diffusion/nemo/post_training/README.md) | Coming soon | +| Changing the Video Tokenizer | Coming soon | [Supported](./autoregressive/nemo/post_training/tokenizer/README.md) | diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/__init__.py new file mode 100644 index 00000000..dac9a4d7 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/README.md new file mode 100644 index 00000000..f181e23b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/README.md @@ -0,0 +1,427 @@ +# Cosmos Autoregressive-based World Foundation Models + +## Table of Contents +- [Getting Started](#getting-started) + - [Set Up Docker Environment](#set-up-docker-environment) + - [Download Checkpoints](#download-checkpoints) +- [Usage](#usage) + - [Model Types](#model-types) + - [Single and Batch Generation](#single-and-batch-generation) + - [Sample Commands](#sample-commands) + - [Base Models (4B/12B)](#base-basepy-4b-and-12b) + - [Video2World Models (5B/13B)](#video2world-video2worldpy-5b-and-13b) + - [Arguments](#arguments) + - [Common Parameters](#common-parameters) + - [Base Specific Parameters](#base-specific-parameters) + - [Video2World Specific Parameters](#video2world-specific-parameters) + - [Safety Features](#safety-features) + +This page details the steps for using the Cosmos autoregressive-based world foundation models. + +## Getting Started + +### Set Up Docker Environment + +Follow our [Installation Guide](../../../INSTALL.md) to set up the Docker environment. All commands on this page should be run inside Docker. + +### Download Checkpoints + +1. Generate a [Hugging Face](https://huggingface.co/settings/tokens) access token. Set the access token to 'Read' permission (default is 'Fine-grained'). + +2. Log in to Hugging Face with the access token: + +```bash +huggingface-cli login +``` + +3. Download the Cosmos model weights from [Hugging Face](https://huggingface.co/collections/nvidia/cosmos-6751e884dc10e013a0a0d8e6): + +```bash +PYTHONPATH=$(pwd) python cosmos1/scripts/download_autoregressive.py --model_sizes 4B 5B 12B 13B +``` + +4. The downloaded files should be in the following structure: + +``` +checkpoints/ +├── Cosmos-1.0-Autoregressive-4B +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Autoregressive-5B-Video2World +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Autoregressive-12B +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Autoregressive-13B-Video2World +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Tokenizer-CV8x8x8 +│ ├── decoder.jit +│ ├── encoder.jit +│ └── mean_std.pt +├── Cosmos-1.0-Tokenizer-DV8x16x16 +│ ├── decoder.jit +│ └── encoder.jit +├── Cosmos-1.0-Diffusion-7B-Decoder-DV8x16x16ToCV8x8x8 +│ ├── aux_vars.pt +│ └── model.pt +└── Cosmos-1.0-Guardrail + ├── aegis/ + ├── blocklist/ + ├── face_blur_filter/ + └── video_content_safety_filter/ +``` + +## Usage + + +### Model Types + +There are two model types available for autoregressive world generation: + +1. **Base**: Supports world generation from image/video input + +* Models: `Cosmos-1.0-Autoregressive-4B` and `Cosmos-1.0-Autoregressive-12B` +* Inference script: [base.py](/cosmos1/models/autoregressive/inference/base.py) + +2. **Video2World**: Supports world generation from image/video input and text input + +* Models: `Cosmos-1.0-Autoregressive-5B-Video2World` and `Cosmos-1.0-Autoregressive-13B-Video2World` +* Inference script: [video2world.py](/cosmos1/models/autoregressive/inference/video2world.py) + +Our models now support video extension up to 33 frames. Starting from either a single image or a 9-frame video input, they can generate the remaining frames to reach the 33-frame length (generating 32 or 24 frames, respectively). + +We have evaluated all eight possible configurations (4 models × 2 vision input types: image or video) using 100 test videos on physical AI topics. Below are the failure rates for each configuration: + +| Model | Image input | Video input (9 frames) | +|:------------------------------------------|:--------------:|:-------------------------:| +| Cosmos-1.0-Autoregressive-4B | 15% | 1% | +| Cosmos-1.0-Autoregressive-5B-Video2World | 7% | 2% | +| Cosmos-1.0-Autoregressive-12B | 2% | 1% | +| Cosmos-1.0-Autoregressive-13B-Video2World | 3% | 0% | + +We define failure cases as videos with severe distortions, such as: + +* Sudden appearance of large unexpected objects +* Video degrading to a single solid color + +Note that the following are not considered failures in our analysis: + +* Static video frames +* Minor object distortions or artifacts + +### Single and Batch Generation + +We support both single and batch video generation. + +For generating a single video, `base` mode requires the input argument `--input_image_or_video_path` (image/video input), while `video2world` mode requires both `--input_image_or_video_path` (image/video input) and `--prompt` (text input). + +Note that our model only works with 1024x640 resolution videos. If the input image/video is not in this resolution, it will be resized and cropped. + +For generating a batch of videos, both `base` and `video2world` require `--batch_input_path` (path to a JSONL file). For `base`, the JSONL file should contain one visual input per line in the following format, where each line must contain a "visual_input" field: + +```json +{"visual_input": "path/to/video1.mp4"} +{"visual_input": "path/to/video2.mp4"} +``` + +For `video2world`, each line in the JSONL file must contain both "prompt" and "visual_input" fields: + +```json +{"prompt": "prompt1", "visual_input": "path/to/video1.mp4"} +{"prompt": "prompt2", "visual_input": "path/to/video2.mp4"} +``` + +### Sample Commands + +There are two main demo scripts for autoregressive world generation: `base.py` and `video2world.py`. Below you will find sample commands for single and batch generation, as well as commands for running with low-memory GPUs using model offloading. We also provide a memory usage table comparing different offloading strategies to help with configuration. + +#### Base (base.py): 4B and 12B + +Generates world from image/video input. + +The `input_type` argument can be either `video` or `image`. We have tuned the sampling parameters `top_p` and `temperature` to achieve the best performance. Please use the provided values in the command examples. + +Note that the command examples below all use video input. If you want to use image input, please change the `input_type` to `image`. + +##### Single Generation + +```bash +# Example using 4B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/base.py \ + --input_type=video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --video_save_name=Cosmos-1.0-Autoregressive-4B \ + --ar_model_dir=Cosmos-1.0-Autoregressive-4B \ + --top_p=0.8 \ + --temperature=1.0 + +# Example for low-memory GPUs using 4B model with model offloading +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/base.py \ + --input_type=video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --video_save_name=Cosmos-1.0-Autoregressive-4B \ + --ar_model_dir=Cosmos-1.0-Autoregressive-4B \ + --top_p=0.8 \ + --temperature=1.0 \ + --offload_guardrail_models \ + --offload_diffusion_decoder \ + --offload_ar_model \ + --offload_tokenizer + +# Example using 12B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/base.py \ + --input_type=video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --video_save_name=Cosmos-1.0-Autoregressive-12B \ + --ar_model_dir=Cosmos-1.0-Autoregressive-12B \ + --top_p=0.9 \ + --temperature=1.0 + +# Example for low-memory GPUs using 12B model with model offloading +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/base.py \ + --input_type=video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --video_save_name=Cosmos-1.0-Autoregressive-12B \ + --ar_model_dir=Cosmos-1.0-Autoregressive-12B \ + --top_p=0.9 \ + --temperature=1.0 \ + --offload_guardrail_models \ + --offload_diffusion_decoder \ + --offload_ar_model \ + --offload_tokenizer +``` + +##### Batch Generation + +```bash +# Example using 4B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/base.py \ + --input_type=video \ + --batch_input_path=cosmos1/models/autoregressive/assets/v1p0/batch_inputs/base.jsonl \ + --video_save_folder=outputs/Cosmos-1.0-Autoregressive-4B \ + --ar_model_dir=Cosmos-1.0-Autoregressive-4B \ + --top_p=0.8 \ + --temperature=1.0 + +# Example using 12B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/base.py \ + --input_type=video \ + --batch_input_path=cosmos1/models/autoregressive/assets/v1p0/batch_inputs/base.jsonl \ + --video_save_folder=outputs/Cosmos-1.0-Autoregressive-12B \ + --ar_model_dir=Cosmos-1.0-Autoregressive-12B \ + --top_p=0.9 \ + --temperature=1.0 +``` + +##### Example Output + +Here is an example output video generated using base.py with image input, using `Cosmos-1.0-Autoregressive-12B`: + + + +The input image used to generate this video can be found in `cosmos1/models/autoregressive/assets/v1p0/input.jpg`. The image is from [BDD dataset](http://bdd-data.berkeley.edu/). + +Here is an example output video generated using base.py with 9-frame video input, using `Cosmos-1.0-Autoregressive-12B`: + + + +The input video used to generate this video can be found in `cosmos1/models/autoregressive/assets/v1p0/input.mp4`. + +##### Inference Time and GPU Memory Usage + +These numbers may vary based on system specifications and are provided for reference only. + +| Offloading Strategy | Cosmos-1.0-Autoregressive-4B | Cosmos-1.0-Autoregressive-12B | +|-------------|---------|---------| +| No offloading | 31.3 GB | 47.5 GB | +| Guardrails | 28.9 GB | 45.2 GB | +| Guardrails & Diffusion decoder | 28.5 GB | 43.1 GB | +| Guardrails & Diffusion decoder & Tokenizer | 27.3 GB | 42.9 GB | +| Guardrails & Diffusion decoder & Tokenizer & AR model | 18.7 GB | 27.4 GB | + +End-to-end inference runtime on one H100 without offloading and after model initialization: + +| Cosmos-1.0-Autoregressive-4B | Cosmos-1.0-Autoregressive-12B | +|---------|---------| +| ~62 seconds | ~119 seconds | + +#### Video2World (video2world.py): 5B and 13B + +Generates world from image/video and text input. + +The `input_type` argument can be either `text_and_video` or `text_and_image`. We have tuned the sampling parameters `top_p` and `temperature` to achieve the best performance. Please use the provided values in the command examples. + +Note that the command examples below all use video input. If you want to use image input, please change the `input_type` to `text_and_image`. + +##### Single Generation + +```bash +# Example using 5B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/video2world.py \ + --input_type=text_and_video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --prompt="A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." \ + --video_save_name=Cosmos-1.0-Autoregressive-5B-Video2World \ + --ar_model_dir=Cosmos-1.0-Autoregressive-5B-Video2World \ + --top_p=0.7 \ + --temperature=1.0 + +# Example for low-memory GPUs using 5B model with model offloading +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/video2world.py \ + --input_type=text_and_video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --prompt="A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." \ + --video_save_name=Cosmos-1.0-Autoregressive-5B-Video2World \ + --ar_model_dir=Cosmos-1.0-Autoregressive-5B-Video2World \ + --top_p=0.7 \ + --temperature=1.0 \ + --offload_guardrail_models \ + --offload_diffusion_decoder \ + --offload_ar_model \ + --offload_tokenizer \ + --offload_text_encoder_model + +# Example using 13B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/video2world.py \ + --input_type=text_and_video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --prompt="A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." \ + --video_save_name=Cosmos-1.0-Autoregressive-13B-Video2World \ + --ar_model_dir=Cosmos-1.0-Autoregressive-13B-Video2World \ + --top_p=0.8 \ + --temperature=1.0 \ + --offload_guardrail_models + +# Example for low-memory GPUs using 13B model with model offloading +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/video2world.py \ + --input_type=text_and_video \ + --input_image_or_video_path=cosmos1/models/autoregressive/assets/v1p0/input.mp4 \ + --prompt="A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." \ + --video_save_name=Cosmos-1.0-Autoregressive-13B-Video2World \ + --ar_model_dir=Cosmos-1.0-Autoregressive-13B-Video2World \ + --top_p=0.8 \ + --temperature=1.0 \ + --offload_guardrail_models \ + --offload_diffusion_decoder \ + --offload_ar_model \ + --offload_tokenizer \ + --offload_text_encoder_model +``` + +##### Batch Generation + +```bash +# Example using 5B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/video2world.py \ + --input_type=text_and_video \ + --batch_input_path=cosmos1/models/autoregressive/assets/v1p0/batch_inputs/video2world.jsonl \ + --video_save_folder=outputs/Cosmos-1.0-Autoregressive-5B-Video2World \ + --ar_model_dir=Cosmos-1.0-Autoregressive-5B-Video2World \ + --top_p=0.7 \ + --temperature=1.0 + +# Example using 13B model +CUDA_VISIBLE_DEVICES=0 PYTHONPATH=$(pwd) python cosmos1/models/autoregressive/inference/video2world.py \ + --input_type=text_and_video \ + --batch_input_path=cosmos1/models/autoregressive/assets/v1p0/batch_inputs/video2world.jsonl \ + --video_save_folder=outputs/Cosmos-1.0-Autoregressive-13B-Video2World \ + --ar_model_dir=Cosmos-1.0-Autoregressive-13B-Video2World \ + --top_p=0.8 \ + --temperature=1.0 \ + --offload_guardrail_models +``` + +##### Example Output + +Here is an example output video generated using video2world.py with image input, using `Cosmos-1.0-Autoregressive-13B-Video2World`: + + + +The input image used to generate this video can be found in `cosmos1/models/autoregressive/assets/v1p0/input.jpg`. The prompt for generating the video is: + +``` +A driving video captures a serene urban street scene on a sunny day. The camera is mounted on the dashboard of a moving vehicle, providing a first-person perspective as it travels down a two-lane road. The street is lined with parked cars on both sides, predominantly black and silver sedans and SUVs. The road is flanked by a mix of residential and commercial buildings, with a prominent red-brick building on the left side, featuring multiple windows and a flat roof. The sky is clear with a few scattered clouds, casting soft shadows on the street. Trees with lush green foliage line the right side of the road, providing a natural contrast to the urban environment. The camera remains steady, maintaining a consistent forward motion, suggesting a leisurely drive. Traffic is light, with a few vehicles moving in the opposite direction, including a black sedan and a yellow taxi. Street signs are visible, including a no-parking sign on the right. The overall atmosphere is calm and peaceful, with no pedestrians visible, emphasizing the focus on the drive and the surrounding urban landscape. +``` + +Here is an example output video generated using video2world.py with 9-frame video input, using `Cosmos-1.0-Autoregressive-13B-Video2World`: + + + +The input video used to generate this video can be found in `cosmos1/models/autoregressive/assets/v1p0/input.mp4`. The prompt for generating the video is: + +``` +A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions. +``` + +##### Inference Time and GPU Memory Usage + +These numbers may vary based on system specifications and are provided for reference only. + +| Offloading Strategy | Cosmos-1.0-Autoregressive-5B-Video2World | Cosmos-1.0-Autoregressive-13B-Video2World | +|-------------|---------|---------| +| No offloading | 66.2 GB | > 80 GB | +| Guardrails | 58.7 GB | 76.6 GB | +| Guardrails & T5 encoder | 41.3 GB | 58.0 GB | +| Guardrails & T5 encoder & Diffusion decoder | 29.0 GB | 46.9 GB | +| Guardrails & T5 encoder & Diffusion decoder & Tokenizer | 28.8 GB | 46.7 GB | +| Guardrails & T5 encoder & Diffusion decoder & Tokenizer & AR model | 21.1 GB | 30.9 GB | + +End-to-end inference runtime on one H100 with no offloading for 5B model and guardrail offloading for 13B, after model initialization: + +| Cosmos-1.0-Autoregressive-5B-Video2World | Cosmos-1.0-Autoregressive-13B-Video2World | +|---------|---------| +| ~73 seconds | ~150 seconds | + +### Arguments + +#### Common Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--checkpoint_dir` | Directory containing model weights | "checkpoints" | +| `--video_save_name` | Output video filename for single video generation | "output" | +| `--video_save_folder` | Folder where all output videos are stored | "outputs/" | +| `--input_image_or_video_path` | Input image or video path. Required for single video generation | None | +| `--batch_input_path` | Folder containing input images or videos. Required for batch video generation | None | +| `--num_input_frames` | Number of input frames to use for Video2World prediction | 9 | +| `--temperature` | Temperature used while sampling | 1.0 (recommend using values in sample commands provided) | +| `--top_p` | Top-p value for top-p sampling | 0.8 (recommend using values in sample commands provided) | +| `--seed` | Random seed | 0 | +| `--disable_diffusion_decoder` | When set to True, use discrete tokenizer to decode discrete tokens to video. Otherwise, use diffusion decoder to decode video | False | +| `--offload_guardrail_models` | Offload guardrail models after inference, used for low-memory GPUs | False | +| `--offload_diffusion_decoder` | Offload diffusion decoder after inference, used for low-memory GPUs | False | +| `--offload_ar_model` | Offload AR model after inference, used for low-memory GPUs | False | +| `--offload_prompt_upsampler` | Offload prompt upsampler after inference, used for low-memory GPUs | False | + +#### Base Specific Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--ar_model_dir` | Directory containing AR model weight | "Cosmos-1.0-Autoregressive-4B" | +| `--input_type` | Input type, either `video` or `image` | "video" | + +#### Video2World Specific Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--ar_model_dir` | Directory containing AR model weight | "Cosmos-1.0-Autoregressive-4B" | +| `--input_type` | Input type, either `text_and_video` or `text_and_image` | "text_and_video" | +| `--prompt` | Text prompt for single video generation. Required for single video generation | None | +| `--input_prompts_path` | Path to JSONL file for batch video generation. Required for batch video generation | None | +| `--offload_text_encoder_model` | Offload text encoder after inference, used for low-memory GPUs | False | + +### Safety Features + +The model uses a built-in safety guardrail system that cannot be disabled. Generating human faces is not allowed and will be blurred by the guardrail. + +For more information, check out the [Cosmos Guardrail Documentation](../guardrail/README.md). diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/2285.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/2285.json new file mode 100644 index 00000000..779e4d42 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/2285.json @@ -0,0 +1,470 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "turn faucet right 55" + ], + "videos": [ + { + "video_path": "videos/test/2285/rgb.mp4" + } + ], + "action": [ + [ + 3.694755885959334e-05, + -2.188859032042501e-06, + 0.0012790152122294242, + -2.8210112014539167e-05, + -0.0046018533948842255, + -6.994484654239804e-06, + 1.0 + ], + [ + 0.010676051978720904, + 0.02100880582033654, + 0.01794003645778153, + -0.1465096458188931, + -0.07351492203295605, + 0.11376468733644193, + 1.0 + ], + [ + 0.0337286923347907, + 0.02676878901993706, + 0.03092682159374319, + -0.22758789264717327, + 0.061155943989451025, + 0.265480532495656, + 1.0 + ], + [ + 0.02618589776341088, + 0.013746149484571387, + 0.02039995685432053, + -0.1494188344540986, + 0.15919041647028576, + 0.3299460092877366, + 1.0 + ], + [ + 0.012643488953262054, + 0.0038062243776331712, + 0.011276627395219426, + -0.18825137407403789, + 0.13215896342880962, + 0.27705196749286176, + 1.0 + ], + [ + -0.004175304901216189, + 0.0027127735403914355, + -0.003078991058387456, + -0.347880405325714, + 0.09330845490563976, + 0.10760548565963729, + 1.0 + ], + [ + -0.0024057994824377028, + 0.010210630656230447, + -0.0011528494171932889, + -0.23974278303232774, + 0.0896640310443008, + -0.0015189742126123958, + 1.0 + ], + [ + -0.012326270613945256, + -0.0021584786652756164, + 0.0005709038823380265, + 0.03371655400927077, + 0.13959212436407073, + -0.10059020198054051, + 1.0 + ], + [ + -0.001981170207573133, + 0.003423977508547292, + -0.003176960642959142, + 0.09050007054464028, + 0.15139030372693107, + -0.03870829318886936, + 1.0 + ], + [ + -0.004540034089117157, + 0.005520172963487343, + -0.015884889236249016, + 0.15428252675835652, + 0.1405835771930671, + -0.03706399870077933, + 1.0 + ], + [ + -0.035440398310796144, + 0.034353515139500065, + 0.004077317349162937, + 0.22192956528302882, + 0.1493889520822572, + -0.6769989476609659, + 1.0 + ], + [ + -0.019292733581275503, + 0.03379137083981197, + -0.0309064338041019, + 0.2956720385846467, + 0.27745918663779895, + 0.07737182674825957, + 1.0 + ], + [ + -0.005181075543443696, + 0.008451720054216995, + -0.0018618563408795514, + 0.19455818394050936, + 0.04905882947086513, + 0.01668960589107708, + 1.0 + ], + [ + 0.005381216264735557, + 0.015380726096418648, + -0.002566765079893937, + 0.04309942580678522, + 0.0042388872777867235, + 0.09020159257543175, + 1.0 + ], + [ + 0.007186031176175356, + -0.00614698061643175, + 0.001101563773955914, + -0.009593827454157728, + -0.01896699908159195, + 0.054499818810296405, + 1.0 + ], + [ + 0.011549250405099746, + -0.0049649580828330115, + 0.002836106962281624, + -0.06429213102112324, + 0.0031778291750390224, + 0.04000998598563486, + 1.0 + ], + [ + 0.007222993919257886, + -0.00826483547832337, + 0.0045249979658009976, + -0.06655536252419943, + -0.000708148883062096, + -0.007611979579570352, + 1.0 + ], + [ + 0.0070145857567987585, + 0.0005666897644498622, + 0.007983396147835012, + -0.11030391649224293, + -0.06608895473553174, + 0.06538881172466689, + 1.0 + ], + [ + 0.0052082055237374315, + -0.0034228324778708247, + 0.00687584459844538, + -0.06372225045929429, + -0.04710536167752597, + 0.024578406458658907, + 1.0 + ], + [ + 0.0017071992084923707, + 0.00030860764526297686, + 0.0043993959435383815, + -0.00579835959275389, + -0.00540630206513693, + 0.002961115041230691, + 1.0 + ], + [ + 0.0014426213627179213, + 0.003369071061537826, + 0.00041467257267436284, + -0.003057903371153386, + 0.0012100535124621415, + 0.006615593024420054, + 1.0 + ], + [ + 0.0013221545603001158, + 0.0026765250398937857, + 0.002603668278954194, + 0.0013040946366107264, + -0.0012944841635619186, + 0.0043152947072437435, + 1.0 + ], + [ + 0.000733665536058688, + -0.0031209570799809448, + 0.0011883148435646834, + 0.03195682088644165, + 0.002858750736607184, + -0.01672090889936187, + 1.0 + ] + ], + "state": [ + [ + 0.2954968214035034, + -0.002963056555017829, + 0.19138140976428986, + 3.139813947030813, + 0.039873380213975906, + -0.015342326834797861, + 1.0 + ], + [ + 0.2954827547073364, + -0.0029629268683493137, + 0.19010193645954132, + 3.1397856705640486, + 0.0444752387702465, + -0.015343518927693367, + 1.0 + ], + [ + 0.3050283193588257, + -0.02415306679904461, + 0.17174293100833893, + 2.9881947954469403, + 0.11749561876058577, + -0.1297250390052795, + 1.0 + ], + [ + 0.3311322331428528, + -0.05900271609425545, + 0.1414981186389923, + 2.7358644922548017, + 0.013184878043830395, + -0.381879061460495, + 1.0 + ], + [ + 0.3475594222545624, + -0.0878869891166687, + 0.12783567607402802, + 2.590411754446574, + -0.26257607340812683, + -0.623802661895752, + 1.0 + ], + [ + 0.35373032093048096, + -0.1035987138748169, + 0.12376514822244644, + 2.455495687323161, + -0.5155699849128723, + -0.8110687136650085, + 1.0 + ], + [ + 0.34972843527793884, + -0.09960127621889114, + 0.1252744197845459, + 2.1191625912957868, + -0.6556864976882936, + -0.8409833908081055, + 1.0 + ], + [ + 0.34143683314323425, + -0.09684033691883087, + 0.13119053840637207, + 1.81615141232545, + -0.6987082362174988, + -0.7399810552597047, + 1.0 + ], + [ + 0.33550769090652466, + -0.09146666526794434, + 0.12155254930257797, + 1.735060365992137, + -0.6258283257484437, + -0.5423598885536193, + 1.0 + ], + [ + 0.3335058093070984, + -0.08725463598966599, + 0.12355080246925354, + 1.7189342101388656, + -0.6041160821914673, + -0.352846473455429, + 1.0 + ], + [ + 0.33098721504211426, + -0.07045303285121918, + 0.12739485502243042, + 1.7789068539910995, + -0.581406831741333, + -0.17966768145561215, + 1.0 + ], + [ + 0.28215792775154114, + -0.07285291701555252, + 0.13531644642353058, + 1.9773414452844342, + 0.05754688009619713, + 0.09795258194208144, + 1.0 + ], + [ + 0.2639961838722229, + -0.05954032763838768, + 0.1796131730079651, + 2.2549331506067, + -0.12145110219717026, + 0.32370471954345703, + 1.0 + ], + [ + 0.2594729959964752, + -0.06517010927200317, + 0.18665504455566406, + 2.445100577669688, + -0.1653294861316681, + 0.35155695676803594, + 1.0 + ], + [ + 0.2661232650279999, + -0.07354404777288437, + 0.19921544194221497, + 2.501273425417491, + -0.2260340452194214, + 0.28339198231697077, + 1.0 + ], + [ + 0.2726331949234009, + -0.06719955056905746, + 0.1963859498500824, + 2.5053766091638288, + -0.24301548302173617, + 0.22669106721878055, + 1.0 + ], + [ + 0.2842640280723572, + -0.06214774772524834, + 0.19408757984638214, + 2.4489823301606854, + -0.2692257761955262, + 0.19526901841163632, + 1.0 + ], + [ + 0.2927067279815674, + -0.056939590722322464, + 0.18756449222564697, + 2.380949469404765, + -0.2638161778450012, + 0.20086857676506045, + 1.0 + ], + [ + 0.30190035700798035, + -0.06110314652323723, + 0.18418778479099274, + 2.297703953581401, + -0.2597609758377075, + 0.1047428473830223, + 1.0 + ], + [ + 0.30902615189552307, + -0.06323298066854477, + 0.1786375343799591, + 2.247868450480052, + -0.24645982682704923, + 0.05161337181925773, + 1.0 + ], + [ + 0.311479389667511, + -0.06673336029052734, + 0.17661431431770325, + 2.2436011751466474, + -0.24537597596645352, + 0.04535707458853721, + 1.0 + ], + [ + 0.3124105632305145, + -0.06911744177341461, + 0.17927002906799316, + 2.241345318155833, + -0.25130268931388855, + 0.04207773134112358, + 1.0 + ], + [ + 0.3137268126010895, + -0.07276846468448639, + 0.18006280064582825, + 2.2436064203553876, + -0.2538774609565735, + 0.03825954720377922, + 1.0 + ], + [ + 0.31519603729248047, + -0.07169589400291443, + 0.17716768383979797, + 2.2723840196901044, + -0.2425617128610611, + 0.05129702761769294, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "2285", + "latent_videos": [ + { + "latent_video_path": "latent_videos/test/2285/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/312.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/312.json new file mode 100644 index 00000000..88559a23 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/312.json @@ -0,0 +1,584 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Move the spatula from above the blue cloth to right side of steel pot." + ], + "videos": [ + { + "video_path": "videos/test/312/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + -1.3057586340075067e-18, + 6.019918929572625e-17, + -8.798749138111388e-19, + 1.0 + ], + [ + -0.0036851795134459973, + 0.0005964594087280908, + 0.008699968842164897, + 0.0016730843965351457, + -0.0344277151472984, + -0.006204485680353569, + 1.0 + ], + [ + 0.014311680140787823, + 0.0008558492018047462, + 0.013364957099887779, + -0.019125393984119627, + -0.022573883450002177, + -0.056123811967506626, + 1.0 + ], + [ + 0.025789572382244667, + 0.003358244615525239, + 0.012694104832143132, + -0.012485671805202417, + -0.008736458639672675, + -0.047468568545928486, + 1.0 + ], + [ + 0.030761352700436416, + 0.004427312344989877, + 0.02227814019763436, + 0.001899596928889922, + -0.008950834679781201, + -0.05792642534809878, + 1.0 + ], + [ + 0.02396223093415951, + 0.007037058569824584, + 0.02195164658424653, + -0.01840563666678819, + 0.006317954477111534, + -0.05189861218597867, + 1.0 + ], + [ + 0.012965951100276442, + 0.0005581197400753106, + 0.026134655599224055, + -0.0030610711698908838, + -0.009047934882182997, + -0.04558240514043799, + 1.0 + ], + [ + 0.0038690044138008227, + -0.0007328246188466693, + 0.011464197965829223, + 0.01515875431353772, + 0.0029535016078663695, + -0.010322899157808312, + 1.0 + ], + [ + 0.000430122271949567, + 0.001122256702319776, + 0.0035007544513766488, + -0.00017684242859230439, + 0.013175434904186318, + 0.0033734959893177733, + 1.0 + ], + [ + -0.003544419631880933, + 0.002662904923230491, + 0.001083032944659464, + 0.0029479187188184213, + 0.004447705269774545, + 0.012168233025675679, + 1.0 + ], + [ + -0.001824966716434567, + -0.0009007444184670198, + -0.0009998789881436358, + 0.0031300414338302257, + 0.00309228716082327, + -0.008036744693692718, + 0.0 + ], + [ + -0.005327922119775354, + -0.0014643898083445837, + -0.004821019470740422, + -0.0013571173607584786, + -0.010638886066777833, + 0.013075335604094428, + 0.0 + ], + [ + 0.0011252159429591157, + -0.006487235257788977, + -0.035783125872482474, + -0.03346390917805088, + 0.04477770085503294, + -0.002083940064016248, + 0.0 + ], + [ + -0.01140718898688949, + -0.040901102093701215, + -0.04409504444135978, + -0.00465273481513913, + 0.04145182451618717, + 0.03779916857145825, + 0.0 + ], + [ + -0.021377580452425625, + -0.042346686650232584, + -0.01094038615348942, + 0.01599524410673955, + -0.032084135738752045, + 0.05073034536687226, + 0.0 + ], + [ + -0.026954741969874803, + -0.03865209840658729, + 0.010133886379968371, + -0.015420757924496434, + -0.06730788783451119, + 0.04584304309626695, + 0.0 + ], + [ + -0.011362658787625442, + -0.021931222256950846, + 0.009582839386142452, + 0.02000447648180545, + -0.02590355537189406, + 0.03432547116748737, + 0.0 + ], + [ + -0.010745750741901157, + -0.029044709990889483, + 0.01466113310177139, + 0.020841900455785375, + -0.013665405241649598, + -0.06817604525257055, + 0.0 + ], + [ + -0.0035176865098615205, + -0.00571823730041179, + 0.018017189514983323, + -0.008514751303691963, + -0.022317528493604043, + -0.001913359601718507, + 0.0 + ], + [ + -0.00014765361532236788, + -0.005581508333685162, + 0.009528908093150195, + 0.03203672900325596, + -0.0254606526051524, + -0.002427213220948259, + 0.0 + ], + [ + 0.00249457996341089, + -0.0004622479684282729, + 0.0019235623488338975, + 0.0032403198508591104, + -0.0027752108133985657, + 0.009138521005500326, + 0.0 + ], + [ + 0.00401806632867145, + 0.001994207861431374, + -0.00017875915393910955, + -0.01449271454720781, + 0.017665567325710993, + 0.004090322076041275, + 0.0 + ], + [ + -0.0055844635471732655, + 0.004793844537720179, + -0.00567547354484359, + -0.05165686242038332, + 0.04807906015637417, + 0.017420020545698692, + 1.0 + ], + [ + 0.004040040689690814, + 0.005272303681738551, + -0.03508590133094734, + -0.03439673583915312, + 0.13699005151514415, + 0.02606861520408746, + 1.0 + ], + [ + 0.004208300531379426, + 0.003936451456414206, + -0.023118166827469984, + 0.017825793632947857, + 0.05132025521678355, + 0.017683208371982236, + 1.0 + ], + [ + -0.000522133152347007, + 0.0017525631341676637, + -0.004630213484155431, + 0.06072262021377042, + -0.06473500130244762, + 0.025487215359571082, + 1.0 + ], + [ + 0.00017833463687986348, + -4.630272404446545e-05, + 0.0012905838475789006, + 0.052381381797205605, + -0.09237304300505067, + 0.0372897165862293, + 1.0 + ], + [ + 0.0009179881515653545, + 0.00048279094160306665, + 0.0015389304030818424, + 0.0019093290084078356, + -0.03869105788028779, + 0.022492821535741725, + 1.0 + ], + [ + -0.00022867041187515203, + 8.300587656743483e-05, + -1.8976070421326382e-07, + -0.0005234282905797688, + -0.0014419317569406933, + 3.776281485291846e-07, + 1.0 + ] + ], + "state": [ + [ + 0.271016389131546, + -0.06008037179708481, + 0.11179866641759872, + 3.1354159625531217, + -0.2416917085647583, + -0.018854450434446335, + 1.0 + ], + [ + 0.271016389131546, + -0.06008037179708481, + 0.11179866641759872, + 3.1354159625531217, + -0.2416917085647583, + -0.018854450434446335, + 1.0 + ], + [ + 0.2695077061653137, + -0.06070222333073616, + 0.10247327387332916, + 3.1356200809874615, + -0.20722194015979764, + -0.012735459953546522, + 1.0 + ], + [ + 0.2862485647201538, + -0.06185116991400719, + 0.09234399348497391, + 3.1047915016585073, + -0.18398559093475342, + 0.04419996216893197, + 1.0 + ], + [ + 0.31404298543930054, + -0.06444861739873886, + 0.08471224457025528, + 3.0835975987189492, + -0.17330308258533478, + 0.09202975779771805, + 1.0 + ], + [ + 0.34851500391960144, + -0.06700261682271957, + 0.06836187839508057, + 3.0755838995152196, + -0.16072475910186768, + 0.15008577704429626, + 1.0 + ], + [ + 0.37656140327453613, + -0.0713273361325264, + 0.051033295691013336, + 3.0488039051466664, + -0.16338630020618436, + 0.202992781996727, + 1.0 + ], + [ + 0.3938298523426056, + -0.0708126649260521, + 0.02751769684255123, + 3.0385050644450864, + -0.1499919891357422, + 0.24804408848285675, + 1.0 + ], + [ + 0.39931178092956543, + -0.06988933682441711, + 0.016745975241065025, + 3.0520710070901593, + -0.15185910463333127, + 0.258739173412323, + 1.0 + ], + [ + 0.4005841016769409, + -0.07103259116411209, + 0.013463610783219337, + 3.052221233146735, + -0.16528300940990448, + 0.2565272152423859, + 1.0 + ], + [ + 0.3980337083339691, + -0.07454351335763931, + 0.012050864286720753, + 3.0571316127949437, + -0.170787513256073, + 0.2446315884590149, + 1.0 + ], + [ + 0.3958992660045624, + -0.07406432181596756, + 0.01264763344079256, + 3.0588374381237706, + -0.1731849014759064, + 0.25302609801292425, + 0.0 + ], + [ + 0.3895716965198517, + -0.07378143817186356, + 0.01634293608367443, + 3.0599130039387425, + -0.16364631056785583, + 0.2389284074306488, + 0.0 + ], + [ + 0.38286784291267395, + -0.06575485318899155, + 0.05119138956069946, + 3.02541375358636, + -0.20810186862945554, + 0.24478177726268766, + 0.0 + ], + [ + 0.35312673449516296, + -0.02604183740913868, + 0.08704860508441925, + 3.027709523337432, + -0.2535407543182373, + 0.2109958827495575, + 0.0 + ], + [ + 0.32233408093452454, + 0.011660892516374588, + 0.08755011111497879, + 3.057719798880168, + -0.22705060243606567, + 0.15554553270339966, + 0.0 + ], + [ + 0.29351866245269775, + 0.04526985436677933, + 0.06848842650651932, + 3.0539233331852635, + -0.1635184586048126, + 0.10363312810659409, + 0.0 + ], + [ + 0.28205108642578125, + 0.06519842892885208, + 0.05532537400722504, + 3.0799500999325, + -0.14060989022254944, + 0.06681831181049348, + 0.0 + ], + [ + 0.27185627818107605, + 0.09266539663076401, + 0.03755892440676689, + 3.0914426689320287, + -0.12245494872331619, + 0.13452231884002686, + 0.0 + ], + [ + 0.2699645161628723, + 0.09726106375455856, + 0.018984949216246605, + 3.0828182009332856, + -0.10006953030824661, + 0.13531811535358426, + 0.0 + ], + [ + 0.2701168656349182, + 0.10234018415212631, + 0.009179111570119858, + 3.114742998155304, + -0.07451023161411285, + 0.13624751567840576, + 0.0 + ], + [ + 0.2726684510707855, + 0.10310427844524384, + 0.007434902247041464, + 3.1186717079305133, + -0.07197818905115128, + 0.12701390683650973, + 0.0 + ], + [ + 0.27687984704971313, + 0.10163633525371552, + 0.007947701960802078, + 3.1044416037672242, + -0.08973237127065659, + 0.12331527471542357, + 0.0 + ], + [ + 0.2714031934738159, + 0.09634275734424591, + 0.013273345306515694, + 3.054159576194831, + -0.138414204120636, + 0.10756205022335052, + 1.0 + ], + [ + 0.270758718252182, + 0.09407204389572144, + 0.04890434443950653, + 3.020940901833125, + -0.27714020013809204, + 0.09321615844964981, + 1.0 + ], + [ + 0.2685125172138214, + 0.09273179620504379, + 0.07258584350347519, + 3.0419039895110807, + -0.3301966786384582, + 0.0812108665704727, + 1.0 + ], + [ + 0.2665790617465973, + 0.09128714352846146, + 0.0769398882985115, + 3.113161478313156, + -0.2681497931480408, + 0.04828548803925515, + 1.0 + ], + [ + 0.26709210872650146, + 0.09132154285907745, + 0.07574190944433212, + -3.107003955292054, + -0.17666055262088776, + 0.007917086593806744, + 1.0 + ], + [ + 0.26827239990234375, + 0.09090159088373184, + 0.07437273114919662, + -3.1013261025124272, + -0.13717524707317352, + -0.013407813385128973, + 1.0 + ], + [ + 0.26804521679878235, + 0.09082168340682983, + 0.07433833926916122, + -3.1018574481182775, + -0.1357344686985016, + -0.013349609449505806, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.04926097393035889, + 0.3258223533630371, + 0.5496259033679962, + 0.5770704746246338, + 0.5779728293418884, + 0.5779728293418884, + 0.5779728293418884, + 0.5779728293418884, + 0.5788743495941162, + 0.5788743495941162, + 0.5788743495941162, + 0.5788743495941162, + 0.5072224140167236, + 0.21446210145950317, + 0.003185570240020752, + 0.0, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0005114078521728516, + 0.0005114078521728516 + ], + "episode_id": "312", + "latent_videos": [ + { + "latent_video_path": "latent_videos/test/312/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/3266.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/3266.json new file mode 100644 index 00000000..beb5650c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/3266.json @@ -0,0 +1,470 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "put cup from anywhere into sink" + ], + "videos": [ + { + "video_path": "videos/test/3266/rgb.mp4" + } + ], + "action": [ + [ + -0.0007326330083548361, + -0.00016754322184926216, + 0.0011176943338618388, + 0.00046179959613248223, + -0.0030271721533362567, + -0.00023432240206731518, + 1.0 + ], + [ + -0.0023152264291718947, + -0.00023511145344613208, + 0.008371568217464863, + -0.00017025227040215265, + -0.0032188107759487383, + 0.002382778144339071, + 1.0 + ], + [ + -0.0031479820264762866, + 0.0008798578666648383, + 0.013645518671305992, + 0.0007563505803436428, + -0.0036748083225268476, + 0.012561600226588348, + 1.0 + ], + [ + -0.004604630897271628, + 0.0029981738547832214, + 0.012794369852380635, + -0.005258936485537489, + 0.0025573657589009055, + 0.014320093948966816, + 1.0 + ], + [ + -0.0018824249252619655, + 0.004565234411142097, + 0.008272712134217715, + -6.852872913270658e-05, + 0.00979617757588051, + -0.008373893785970756, + 1.0 + ], + [ + -0.004843297855305156, + 0.004486026574118139, + 0.013132809484540195, + 0.002308652262693833, + 0.007935731435575378, + -0.018270184658072806, + 1.0 + ], + [ + 0.001018042139226023, + 0.0012578965644707073, + 0.008146034476846625, + -0.01670921104398916, + 0.05631717331772656, + -0.053897399498263346, + 1.0 + ], + [ + 0.006598026848940242, + 0.0011845640545525909, + 0.000827623194964791, + -0.006911157693852161, + 0.04295871215167792, + -0.01672732649317138, + 1.0 + ], + [ + 0.014296659168862658, + 0.0009990954842583276, + -0.009841603856358778, + 0.011738936016981892, + 0.02143912769821807, + -0.0051705435667975174, + 1.0 + ], + [ + 0.010902939936434803, + 0.0010811959990317756, + -0.007280335451457056, + 0.019304705667351763, + -0.005408602735867421, + -0.0204003432700352, + 1.0 + ], + [ + 0.003790625776839835, + 0.005160169338798126, + 0.0016686302242591904, + 0.01081470448526685, + -0.02266229310321449, + -0.013009007680405587, + 0.0 + ], + [ + 0.00011484382777708856, + 0.0037416484045991526, + 0.0022576401144436386, + 0.006984588651038514, + -0.00248552314062158, + -0.00396882482195083, + 0.0 + ], + [ + 0.001050155791230984, + 0.006077738074337243, + 0.00025883120338362846, + 0.0030688250726914404, + -0.011139542432925255, + 0.00790082036544059, + 0.0 + ], + [ + -0.0028125766755805323, + 0.00011410095302349149, + -0.003184014038301367, + -0.001153905290096619, + 0.0026156077386375915, + 0.00818232018728185, + 0.0 + ], + [ + -0.012442827956436107, + -0.016087819137458725, + -0.01970748986247242, + 0.04298891821672249, + -0.033885791069873696, + 0.029512171157223235, + 0.0 + ], + [ + -0.005539455162833835, + -0.020669329223933195, + -0.03272041121317737, + 0.028297961339808737, + -0.006605512495278495, + 0.028139332735705323, + 0.0 + ], + [ + 0.0061159095379947495, + -0.013703319871067002, + -0.02832528186964502, + 0.007757477956779777, + 0.02709092416201737, + 0.037039977489483046, + 0.0 + ], + [ + 0.020152367104380797, + -0.0038603286883022105, + -0.01771927631289184, + -0.0016445299379208903, + 0.02038997528447429, + 0.0609190445428429, + 0.0 + ], + [ + 0.033093101140414236, + -0.0016466681234564792, + -0.011968057015156928, + -0.011899482171080784, + 0.009640890288878696, + 0.10201348603188172, + 0.0 + ], + [ + 0.03626899575503109, + -0.006159803940588596, + -0.00827674776129196, + -0.029204918052817042, + 0.0038312830550087523, + 0.1034101215879261, + 0.0 + ], + [ + 0.02023654187684149, + -0.005444010415101585, + -0.005391198839010221, + -0.02981765617972912, + 0.005848005447181435, + 0.04036733436886654, + 0.0 + ], + [ + 0.004722980330871796, + 0.0025506528401849178, + 0.009305693756029363, + -0.021062294576465208, + 0.021579815373808913, + -0.006563416193826717, + 0.0 + ], + [ + -0.005496167974590717, + 0.003407848711682173, + 0.01696315500901566, + -0.01964927900176218, + -0.01706635993489661, + -0.015412146344062688, + 0.0 + ] + ], + "state": [ + [ + 0.11536289751529694, + -0.037388790398836136, + 0.10222749412059784, + 3.1116191280358514, + 0.14516817033290863, + -0.1003223732113838, + 1.0 + ], + [ + 0.11449351161718369, + -0.03716662898659706, + 0.10122306644916534, + 3.1121017785095653, + 0.14820100367069244, + -0.10017728805541992, + 1.0 + ], + [ + 0.11098353564739227, + -0.03682572767138481, + 0.09328187257051468, + 3.111561535047837, + 0.15134769678115842, + -0.10268256813287734, + 1.0 + ], + [ + 0.10571416467428207, + -0.0375787615776062, + 0.08029916137456894, + 3.1103873812877616, + 0.15463143587112427, + -0.11550173163414001, + 1.0 + ], + [ + 0.09886128455400467, + -0.04020238295197487, + 0.0684652179479599, + 3.1029139776998242, + 0.1516127586364746, + -0.129900187253952, + 1.0 + ], + [ + 0.095172218978405, + -0.04464365541934967, + 0.06075233593583107, + 3.104179510968276, + 0.14214187860488892, + -0.12106503546237946, + 1.0 + ], + [ + 0.08799020200967789, + -0.04878067970275879, + 0.0486132949590683, + 3.1091458221250257, + 0.13487043976783752, + -0.10234127938747405, + 1.0 + ], + [ + 0.08775471150875092, + -0.05028599873185158, + 0.04044903442263603, + 3.0999446493857583, + 0.08012164384126665, + -0.04654983431100845, + 1.0 + ], + [ + 0.09420560300350189, + -0.05180582031607628, + 0.03914586454629898, + 3.094483973579951, + 0.03788292407989501, + -0.028050806373357773, + 1.0 + ], + [ + 0.10884535312652588, + -0.0527513287961483, + 0.04847504198551178, + 3.1064468567543706, + 0.01671034842729569, + -0.021876789629459385, + 1.0 + ], + [ + 0.11984830349683762, + -0.05381698161363602, + 0.05560566857457161, + 3.126095780292996, + 0.022828992456197735, + -0.001674168393947184, + 1.0 + ], + [ + 0.12359299510717392, + -0.05900866538286209, + 0.053931087255477905, + 3.137196943751075, + 0.04568831995129585, + 0.010991975665092468, + 0.0 + ], + [ + 0.12364659458398819, + -0.06275983899831772, + 0.05168700963258743, + -3.138822779180952, + 0.04819090664386749, + 0.01495442446321249, + 0.0 + ], + [ + 0.12477301061153412, + -0.06882067024707794, + 0.0513610765337944, + -3.136133645317056, + 0.05935079604387284, + 0.007071099244058133, + 0.0 + ], + [ + 0.12215520441532135, + -0.06897066533565521, + 0.05470564588904381, + -3.137774676438891, + 0.05677789822220801, + -0.0011385796824470162, + 0.0 + ], + [ + 0.11087249219417572, + -0.052945349365472794, + 0.07514865696430206, + -3.096459249156066, + 0.09075152128934862, + -0.03062555938959122, + 0.0 + ], + [ + 0.10899090021848679, + -0.03370671346783638, + 0.10913198441267014, + -3.0707103331857404, + 0.09858451783657074, + -0.05857320874929428, + 0.0 + ], + [ + 0.11862091720104218, + -0.02258855663239956, + 0.13761278986930847, + -3.0668152590566358, + 0.07410971820354462, + -0.09753336757421494, + 0.0 + ], + [ + 0.14019174873828888, + -0.022161293774843216, + 0.15402953326702118, + -3.0732048620753964, + 0.05818111076951027, + -0.15990105271339417, + 0.0 + ], + [ + 0.17363065481185913, + -0.02671860344707966, + 0.16413743793964386, + -3.0914151427620133, + 0.05521627888083458, + -0.26248964667320246, + 0.0 + ], + [ + 0.21054963767528534, + -0.030697938054800034, + 0.17069803178310394, + -3.126597611355134, + 0.056271594017744064, + -0.36612620949745184, + 0.0 + ], + [ + 0.23162218928337097, + -0.03303422033786774, + 0.17502345144748688, + 3.124480679752775, + 0.05098330229520798, + -0.40662872791290283, + 0.0 + ], + [ + 0.23444931209087372, + -0.03720172122120857, + 0.165534108877182, + 3.1037682128422937, + 0.029517728835344308, + -0.3996955454349518, + 0.0 + ], + [ + 0.227356418967247, + -0.03859858959913254, + 0.14888149499893188, + 3.0845535044842443, + 0.04715146124362945, + -0.3849256634712219, + 0.0 + ] + ], + "continuous_gripper_state": [ + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.006563663482666016, + 0.20585864782333374, + 0.491834819316864, + 0.7382865846157074, + 0.853794664144516, + 0.8879803046584129, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557, + 0.8883927315473557 + ], + "episode_id": "3266", + "latent_videos": [ + { + "latent_video_path": "latent_videos/test/3266/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/43.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/43.json new file mode 100644 index 00000000..d0b2ea5e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/43.json @@ -0,0 +1,470 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "pick up bowl and put in small4fbox" + ], + "videos": [ + { + "video_path": "videos/test/43/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + 1.5071847976197504e-17, + 3.8568143366312044e-18, + 1.3312403053503924e-18, + 1.0 + ], + [ + 0.0025053845543873336, + 0.0009974226428879547, + 0.018047173990091835, + 0.023831050266946024, + -0.023100069461295655, + 0.03488260477627205, + 1.0 + ], + [ + 0.025875692032972304, + -0.006139807196137211, + 0.0054616739864492584, + 0.040502562993146855, + 0.11780547326858164, + 0.15946217804222154, + 1.0 + ], + [ + 0.024032211027643614, + -0.008373489472863524, + 0.012241485794181207, + 0.08444473992836561, + 0.13082070208459048, + 0.10832395548628035, + 1.0 + ], + [ + 0.013176007515266248, + -0.0075489789460890045, + 0.017759933780408563, + 0.06697791660168664, + 0.08798762037331842, + 0.11348153023280193, + 1.0 + ], + [ + 0.0058657890355108275, + -0.017164415314878823, + 0.029618716522731535, + 0.016152509757330014, + 0.044191464346175297, + 0.07355056903502373, + 1.0 + ], + [ + -0.000478839537224607, + -0.012363837359651462, + 0.022124868025403158, + -0.016542896242652316, + 0.04457698771765512, + 0.05286920041510412, + 1.0 + ], + [ + -0.005940218256592387, + -0.006050556364383413, + 0.025120823429308332, + -0.0646673147778959, + 0.04267448746464814, + 0.0022565940020201724, + 1.0 + ], + [ + 3.2974928483263255e-05, + -0.0007166274828749894, + 0.008956290608860013, + -0.0438533820789159, + 0.04415993049542429, + -0.013362271040647876, + 1.0 + ], + [ + -0.009070370611783144, + 0.006382589415038583, + 0.01357031954240243, + -0.016562923232955416, + -0.009495583467064096, + 0.005408429075672425, + 1.0 + ], + [ + -0.001315345624934077, + 0.00325833535690746, + -0.00016927218735777384, + -0.013456412355049739, + -0.010777246212272057, + -0.023205991265735616, + 0.0 + ], + [ + -0.003794499064591688, + 0.006831322095418517, + -0.0009550853755016471, + -0.033899071424333425, + -0.03382685469319165, + -0.02848710087425441, + 0.0 + ], + [ + 0.0006281074064607681, + 0.015450752247307263, + -0.009241054928844302, + 0.004014327509893675, + -0.05606556453847002, + -0.017057405589730026, + 0.0 + ], + [ + -0.0065618579198078834, + 0.02652249694787712, + -0.012263536127494565, + 0.002314913914480741, + -0.11751298009107754, + -0.08446074436192404, + 0.0 + ], + [ + 0.005643400887003247, + 0.014513786139821872, + -0.03183443511712733, + -0.001409704971010444, + -0.05478928041853172, + -0.10351697191230107, + 0.0 + ], + [ + 0.0026908138699559448, + 0.016296480221728166, + -0.044865280569097506, + 0.025835512023336702, + -0.03465785036615068, + -0.048592414891233095, + 0.0 + ], + [ + -0.0014893910064430551, + -0.005595953014663707, + -0.037600168122311775, + 0.07211683222745922, + -0.06286863123456594, + -0.02498886667331872, + 0.0 + ], + [ + -0.005333408443971233, + -0.024720120796234608, + -0.027392934750330396, + 0.07683181665092476, + -0.0688186969386123, + -0.05034974639872532, + 0.0 + ], + [ + -0.006733201488969607, + -0.033625013517657586, + -0.024454443095176876, + 0.0895370265053668, + -0.0291224984700415, + -0.029576457528306756, + 0.0 + ], + [ + -0.022141177864657868, + -0.036885636159547354, + -0.0026864722529761865, + 0.003619637991749144, + -0.031515645624830564, + -0.02849824160596472, + 0.0 + ], + [ + -0.01838824729819141, + -0.05723653545959213, + 0.007147630242036668, + -0.07384679806913803, + -0.023608767837170284, + -0.054815573884277775, + 0.0 + ], + [ + -0.0071090429839975496, + -0.023747885778691434, + 0.024865220430972126, + 0.017667652662575476, + 0.021955578388497467, + 0.01820143238019961, + 0.0 + ], + [ + 0.00048507620349725955, + -0.025891931601314845, + 0.02822314844016797, + 0.028079391306688562, + 0.02720996813838235, + -0.03163831297947805, + 1.0 + ] + ], + "state": [ + [ + 0.3124028146266937, + -0.1670023649930954, + 0.13412229716777802, + -3.058987693982669, + -0.01706465147435665, + 0.0422857403755188, + 1.0 + ], + [ + 0.3124028146266937, + -0.1670023649930954, + 0.13412229716777802, + -3.058987693982669, + -0.01706465147435665, + 0.0422857403755188, + 1.0 + ], + [ + 0.31519269943237305, + -0.16638880968093872, + 0.11609974503517151, + -3.034618640440055, + 0.008843555115163326, + 0.00943705253303051, + 1.0 + ], + [ + 0.34096089005470276, + -0.1594574749469757, + 0.11109619587659836, + -2.996455418067523, + -0.09147799760103226, + -0.16143859922885895, + 1.0 + ], + [ + 0.367180734872818, + -0.15353913605213165, + 0.1024353951215744, + -2.9001896550231656, + -0.20456172525882718, + -0.2893754243850708, + 1.0 + ], + [ + 0.38585492968559265, + -0.14702101051807404, + 0.08999361097812653, + -2.806879552202769, + -0.26115173101425176, + -0.424908310174942, + 1.0 + ], + [ + 0.40696629881858826, + -0.12809979915618896, + 0.06992894411087036, + -2.769095154600688, + -0.27781641483306885, + -0.5122130513191223, + 1.0 + ], + [ + 0.42000612616539, + -0.11298245936632156, + 0.05430740490555763, + -2.767312141256877, + -0.2994966506958008, + -0.5807027816772461, + 1.0 + ], + [ + 0.4286109507083893, + -0.10090795159339905, + 0.03232754021883011, + -2.8261442502313336, + -0.3383412957191468, + -0.5994630455970763, + 1.0 + ], + [ + 0.4328611195087433, + -0.0996217355132103, + 0.024516554549336433, + -2.8693188448720655, + -0.3844661712646484, + -0.6005479097366332, + 1.0 + ], + [ + 0.4290880560874939, + -0.10006532073020935, + 0.007407204248011112, + -2.8848002572828015, + -0.37386453151702886, + -0.6034003496170044, + 1.0 + ], + [ + 0.4264667332172394, + -0.10213843733072281, + 0.006308846175670624, + -2.9081851114803037, + -0.36920967698097223, + -0.5763988494873047, + 0.0 + ], + [ + 0.4199540615081787, + -0.10609526932239532, + 0.004332495853304863, + -2.955712007480212, + -0.34264570474624634, + -0.5386803150177001, + 0.0 + ], + [ + 0.4100002944469452, + -0.11982583999633789, + 0.010407963767647743, + -2.9609848727756223, + -0.2905698418617248, + -0.5103985667228699, + 0.0 + ], + [ + 0.38886758685112, + -0.14042077958583832, + 0.01552189327776432, + -2.988737003999301, + -0.1885402947664261, + -0.40486007928848267, + 0.0 + ], + [ + 0.3813638389110565, + -0.15808483958244324, + 0.04531457647681236, + -3.0117819627099713, + -0.14895971119403842, + -0.2930799424648285, + 0.0 + ], + [ + 0.37154391407966614, + -0.1780669242143631, + 0.08762290328741074, + -2.9939189871125897, + -0.12067330628633499, + -0.24004848301410675, + 0.0 + ], + [ + 0.36566340923309326, + -0.1766248345375061, + 0.12518146634101868, + -2.925690682726451, + -0.062096185982227325, + -0.2060667276382446, + 0.0 + ], + [ + 0.3622457683086395, + -0.15723751485347748, + 0.156841441988945, + -2.852751465635844, + -0.00552804209291935, + -0.14224748313426971, + 0.0 + ], + [ + 0.3589819669723511, + -0.13124585151672363, + 0.18982292711734772, + -2.7634905000501355, + 0.013969238847494125, + -0.10560649633407593, + 0.0 + ], + [ + 0.34069907665252686, + -0.09583279490470886, + 0.20624396204948425, + -2.7594295759969434, + 0.032731253653764725, + -0.06747594475746155, + 0.0 + ], + [ + 0.32660338282585144, + -0.038979966193437576, + 0.22155097126960754, + -2.831927152472087, + 0.03415024280548096, + -0.007780597545206547, + 0.0 + ], + [ + 0.3191721439361572, + -0.008725471794605255, + 0.20535770058631897, + -2.8150951286130628, + 0.01877525262534618, + -0.03180790692567826, + 0.0 + ], + [ + 0.3203788995742798, + 0.024829164147377014, + 0.18692393600940704, + -2.7865679581933698, + -0.017147859558463097, + -0.010578283108770849, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.009293973445892334, + 0.12860965728759766, + 0.4250529408454895, + 0.7041586935520172, + 0.8747213333845139, + 0.9139323532581329, + 0.9107357934117317, + 0.9048766940832138, + 0.9029988870024681, + 0.9022414609789848, + 0.9022414609789848, + 0.9018614068627357, + 0.9018614068627357, + 0.9022414609789848, + 0.8350401669740677 + ], + "episode_id": "43", + "latent_videos": [ + { + "latent_video_path": "latent_videos/test/43/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/570.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/570.json new file mode 100644 index 00000000..e2732a0c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/test/570.json @@ -0,0 +1,584 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "put spatula on cutting board" + ], + "videos": [ + { + "video_path": "videos/test/570/rgb.mp4" + } + ], + "action": [ + [ + -4.722467804532301e-05, + -0.0003341993399766763, + -3.57072718622728e-07, + 0.0021074036615876548, + -0.00029870959237146067, + 0.00041899800845209825, + 1.0 + ], + [ + -0.004733307982038246, + -0.0025737790496467654, + 0.013803669869593244, + 0.02220713994383716, + -0.02927230422338475, + -0.005248142609315969, + 1.0 + ], + [ + -0.009172480323429336, + -0.019021068680525288, + 0.007754635237001917, + -0.007861684495188388, + 0.01770332435246729, + -0.06951767539486318, + 1.0 + ], + [ + 0.0069614975405418505, + -0.02402953570362439, + 0.004413591235709125, + 0.02723589751666945, + 0.0067801636710528084, + -0.18191026580336347, + 1.0 + ], + [ + 0.010134963349882899, + -0.020443623345868002, + 0.01844668618657973, + 0.015912803426351995, + 0.024425563656219115, + -0.12808894442963878, + 1.0 + ], + [ + 0.002023928007080345, + -0.009891347179262686, + 0.018361485339941433, + 0.011642213034042382, + 0.004490688772247499, + -0.02575820913723037, + 1.0 + ], + [ + -0.0009430748114383215, + -0.02389138928120018, + 0.02746441917318989, + -0.006138359297392099, + -0.02053582861227795, + -0.04776357030259132, + 1.0 + ], + [ + 0.0038951429041164747, + -0.006760159770276475, + 0.016616090109706223, + -0.007630784978491057, + 0.027280854127543882, + -0.0496863623082471, + 1.0 + ], + [ + -0.0022182905468117516, + 0.002795962966616956, + 0.013023661924876286, + 0.005478670006070614, + 0.007484440375675257, + -0.04098408288921299, + 1.0 + ], + [ + -0.0032049139029118427, + 0.004824539391468373, + 0.012787408818617183, + 0.02035864315728666, + -0.012549844849776424, + -0.02483840574131158, + 1.0 + ], + [ + -0.0002837162831980694, + 0.005241212996970935, + 0.0007884250976355273, + -0.023340337508688264, + 0.03309498943621352, + -0.01597172814397185, + 1.0 + ], + [ + -0.0015181683477924802, + -0.0006732087880102378, + -8.697541597353662e-06, + 0.004190091005527802, + -0.009598281565836097, + 0.005757837833194817, + 1.0 + ], + [ + 0.0004951797932638662, + -0.0033225824018652942, + 0.00012710775025278, + 0.011680608644932908, + -0.01478652154224407, + 0.00803111880338693, + 0.0 + ], + [ + 0.0008486036939572637, + -0.0022078446128982, + -0.002363661390999024, + 0.019413312374755885, + -0.01844835920459879, + 0.026449305679687986, + 0.0 + ], + [ + 0.010433964966636039, + -0.0004288411955233294, + -0.019281338736778942, + -0.027269831950388235, + -0.019215984665781373, + 0.055524399211561666, + 0.0 + ], + [ + 0.020835961959476016, + -0.012143695109688706, + -0.013326729511576254, + -0.03288723360334419, + -0.03281799898778154, + 0.05181242153601606, + 0.0 + ], + [ + 0.018273479883349347, + -0.023036429592585953, + -0.0003571539411189626, + 0.007764500928410775, + -0.001971639870559572, + 0.047523094418235184, + 0.0 + ], + [ + 0.009729952604813581, + -0.022222341994101304, + 0.009297850191776423, + 0.0063162678377387356, + 0.02967875358785983, + 0.0815949699883576, + 0.0 + ], + [ + -0.00445725780930581, + -0.021323309131352525, + 0.009026547800937243, + -0.013297325880704581, + 0.023032414000901547, + 0.06740470542543751, + 0.0 + ], + [ + 0.0028410842834708672, + -0.014981662678625364, + 0.0016711832023542483, + 0.023534051244100893, + 0.021163675073214257, + 0.02503609621055804, + 0.0 + ], + [ + -0.005657642981929401, + -0.007297073165695441, + 0.006554255649753995, + 0.0071327744772359284, + 0.0023059055633151927, + 0.03178008794239983, + 0.0 + ], + [ + 0.001921243474120459, + -0.003389679962287874, + -0.0002536067793392802, + 0.019048437912454762, + 0.00971359648055152, + 0.004829277288766817, + 0.0 + ], + [ + 0.0026557205102495916, + -0.004434606215870695, + 0.00010455113026990841, + 0.030956151006554498, + 0.018124485810964385, + 0.030862210901670975, + 0.0 + ], + [ + -0.005809696994351354, + 0.002265192834887688, + -0.0013981661358099596, + 0.0143516620658622, + -0.0381769336382508, + 0.008179665421595874, + 0.0 + ], + [ + 0.0013194264971575099, + 0.007812764744974466, + -0.019764746331861233, + -0.032156935623909005, + -0.013370988870379038, + -0.010233082853672042, + 1.0 + ], + [ + -0.005689952101060145, + 0.010373210232277694, + -0.028340677841996866, + -0.02919215387105963, + 0.004710378768886177, + 0.022329458885177646, + 1.0 + ], + [ + -0.0017510513472257315, + 0.01601011962627829, + -0.01875134350246353, + -0.045763929730332414, + 0.0009226330313950694, + 0.024032723222642736, + 1.0 + ], + [ + -0.002319946281722921, + 0.007805966689515878, + -0.0076771603999764935, + -0.02438047295730655, + -0.0021454601960541266, + 0.06379262123769748, + 1.0 + ], + [ + -6.705572553488375e-05, + 0.00040305975896096915, + -5.287951913568087e-07, + -0.0025417823245957234, + -0.0004231548452774075, + -0.00013058426245359927, + 1.0 + ] + ], + "state": [ + [ + 0.3559815585613251, + -0.03795568272471428, + 0.13810071349143982, + 3.136511311987885, + -0.0968388170003891, + 0.5206893086433411, + 1.0 + ], + [ + 0.3557746410369873, + -0.03768909350037575, + 0.13809481263160706, + 3.1386595642012587, + -0.09654223173856735, + 0.5202668309211731, + 1.0 + ], + [ + 0.3515825569629669, + -0.03717122599482536, + 0.12389171123504639, + -3.1228185464912137, + -0.06725337356328964, + 0.5254385471343994, + 1.0 + ], + [ + 0.33448317646980286, + -0.024934114888310432, + 0.1158958300948143, + -3.1353844498568257, + -0.08609630912542345, + 0.5948574542999268, + 1.0 + ], + [ + 0.32705020904541016, + -0.0009176631574518979, + 0.11224591732025146, + -3.1238708571814975, + -0.09257516264915466, + 0.777508020401001, + 1.0 + ], + [ + 0.3208649456501007, + 0.022134307771921158, + 0.09517873823642731, + -3.1199477539681872, + -0.11850351095199585, + 0.9060081243515015, + 1.0 + ], + [ + 0.3153320550918579, + 0.031749896705150604, + 0.07740215957164764, + -3.1113684979551515, + -0.1235113739967346, + 0.9318597912788391, + 1.0 + ], + [ + 0.2969019114971161, + 0.04837794229388237, + 0.05075971037149429, + -3.1235241313749036, + -0.10428294539451599, + 0.9804770350456238, + 1.0 + ], + [ + 0.2941492199897766, + 0.056951798498630524, + 0.03476352244615555, + -3.136333102947571, + -0.13232974708080292, + 1.030077338218689, + 1.0 + ], + [ + 0.2962411940097809, + 0.05513753741979599, + 0.021546635776758194, + -3.136312846681573, + -0.13991791009902954, + 1.0714246034622192, + 1.0 + ], + [ + 0.29975318908691406, + 0.05164218321442604, + 0.008412182331085205, + -3.119456673162528, + -0.12745577096939087, + 1.0965306758880615, + 1.0 + ], + [ + 0.3043232262134552, + 0.04910905286669731, + 0.0074792117811739445, + 3.1384360470161816, + -0.16088122129440308, + 1.1119564771652222, + 1.0 + ], + [ + 0.30305543541908264, + 0.048062726855278015, + 0.0072425068356096745, + -3.1396213664026043, + -0.15129844844341278, + 1.1061017513275146, + 1.0 + ], + [ + 0.30031248927116394, + 0.0500057078897953, + 0.0071979607455432415, + -3.126723226281353, + -0.13649125397205353, + 1.0980255603790283, + 0.0 + ], + [ + 0.29861268401145935, + 0.051453154534101486, + 0.009687370620667934, + -3.1037269217246255, + -0.11760479211807251, + 1.0716760158538818, + 0.0 + ], + [ + 0.3027540445327759, + 0.05842006951570511, + 0.030062178149819374, + -3.1245949381315192, + -0.09612500667572021, + 1.0166760683059692, + 0.0 + ], + [ + 0.3028514087200165, + 0.08122190833091736, + 0.04553069546818733, + 3.13061598979915, + -0.062305301427841187, + 0.9653578400611877, + 0.0 + ], + [ + 0.29428353905677795, + 0.10932482779026031, + 0.04677256569266319, + 3.141357430203822, + -0.06078473478555679, + 0.9177280068397522, + 0.0 + ], + [ + 0.28288042545318604, + 0.1309887021780014, + 0.038077738136053085, + -3.1305399100580296, + -0.09027984738349915, + 0.8358418345451355, + 0.0 + ], + [ + 0.26454320549964905, + 0.1426478773355484, + 0.02892136201262474, + -3.1377259780471256, + -0.112358920276165, + 0.7677741646766663, + 0.0 + ], + [ + 0.25629478693008423, + 0.1555175483226776, + 0.02763684280216694, + -3.111351067321845, + -0.1333899646997452, + 0.7424368858337402, + 0.0 + ], + [ + 0.24771831929683685, + 0.15781591832637787, + 0.02061002515256405, + -3.0999616441601, + -0.13466604053974152, + 0.710310697555542, + 0.0 + ], + [ + 0.24692007899284363, + 0.16158251464366913, + 0.02125886082649231, + -3.0802023616903504, + -0.14416837692260742, + 0.7050267457962037, + 0.0 + ], + [ + 0.24603071808815002, + 0.16664598882198334, + 0.0218063835054636, + -3.0446233471208295, + -0.16029149293899536, + 0.6727007627487183, + 0.0 + ], + [ + 0.24288778007030487, + 0.1610865294933319, + 0.02203638106584549, + -3.0294872765713414, + -0.12150047719478607, + 0.6682278513908385, + 0.0 + ], + [ + 0.24831080436706543, + 0.15265926718711853, + 0.04082460701465607, + -3.063064048188277, + -0.10934988409280777, + 0.6799617409706116, + 1.0 + ], + [ + 0.24948430061340332, + 0.13745002448558807, + 0.06747934222221375, + -3.0897903722995004, + -0.11226607859134673, + 0.657188355922699, + 1.0 + ], + [ + 0.2568797767162323, + 0.12173508107662201, + 0.08506770431995392, + -3.132858434813567, + -0.11191060394048691, + 0.632988691329956, + 1.0 + ], + [ + 0.2589930593967438, + 0.11352114379405975, + 0.09236971288919449, + 3.1330881920927247, + -0.10898006707429886, + 0.5688363313674927, + 1.0 + ], + [ + 0.2591536343097687, + 0.11314543336629868, + 0.09236635267734528, + 3.1305325162881097, + -0.10855581611394882, + 0.5689640641212462, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.03792673349380493, + 0.3258223533630371, + 0.6472503244876862, + 0.7018977999687195, + 0.6763874590396881, + 0.6700286269187927, + 0.6700286269187927, + 0.6700286269187927, + 0.6700286269187927, + 0.6700286269187927, + 0.6700286269187927, + 0.6700286269187927, + 0.6107503175735474, + 0.29881423711776733, + 0.024030208587646484, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "570", + "latent_videos": [ + { + "latent_video_path": "latent_videos/test/570/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/13109.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/13109.json new file mode 100644 index 00000000..0a54731f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/13109.json @@ -0,0 +1,470 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "put corn in pot which is in sink distractors" + ], + "videos": [ + { + "video_path": "videos/train/13109/rgb.mp4" + } + ], + "action": [ + [ + -0.0016356673592290729, + 0.0014232304994348016, + 0.006595984765087515, + -0.012896675920537913, + -0.008180936561036986, + -0.0013835039064105106, + 1.0 + ], + [ + -0.0035398280017079876, + 0.010369752282656398, + 0.02173316443598503, + -0.05604561735284274, + 0.009147929229343087, + 0.007792697719350199, + 1.0 + ], + [ + -0.0011731187813496637, + 1.2592015392308685e-05, + 0.0053356808398665, + -0.019349668419898616, + 0.04380180911532414, + 0.02703493118696499, + 1.0 + ], + [ + -0.005410624139176912, + 0.0008184797364707952, + 0.008046238190432055, + -0.04095159899875164, + 0.0052706686808839005, + 0.016790457392198057, + 1.0 + ], + [ + 0.0005039070493123949, + -0.0019261692726102802, + 0.0008237299188180624, + -0.00562173135638051, + 0.009667876512424036, + 0.01518791445394935, + 1.0 + ], + [ + -0.0004342795200835655, + -0.001160103669871699, + 0.0038936347597218314, + -0.0031609946189465003, + 0.007711061897496055, + 0.0009621290727139707, + 1.0 + ], + [ + 0.0010225044309666257, + -0.0023854193725248154, + -0.002774478737353711, + 0.004222897798547527, + 0.0015796043720488202, + 0.0023516487259536934, + 1.0 + ], + [ + 5.1615324496354754e-05, + -0.00047877552771801606, + -0.0006640506097102578, + -0.0009371757905454425, + -0.0025830684157682813, + 0.003996495528080935, + 0.0 + ], + [ + -0.0011060859627663696, + -0.0007378191708952141, + 0.0001349672183101587, + 0.006337917091578068, + -0.010665160240759895, + 0.001734081862424481, + 0.0 + ], + [ + 0.001487999363913928, + 0.0028378433005833577, + -0.0003302794691745963, + 9.192192135582545e-05, + -0.0438047713024828, + -0.008470870764521915, + 0.0 + ], + [ + 0.003548333584618942, + 0.009572735277082524, + -0.0010939105473163368, + -0.017056127801969488, + -0.07175386091675678, + -0.024610099664071766, + 0.0 + ], + [ + 0.008157233648524, + 0.005554423088291022, + -0.01593852471953764, + 0.0336497928559805, + -0.022733584951334217, + -0.023125546943136198, + 0.0 + ], + [ + 0.006128332076559636, + 0.0060185371499974485, + -0.020685890536765988, + 0.0534493065296847, + -0.014919948360701303, + -0.014869015587929935, + 0.0 + ], + [ + 8.59549669773982e-05, + 0.012790448511725611, + -0.0005747750415040615, + -0.0034105318461758126, + -0.03374946494865484, + -0.038190886290830964, + 0.0 + ], + [ + 0.002203924322299712, + 0.013120633249322406, + -0.017492622545177302, + 0.024918609007642492, + 0.024919249251260347, + -0.062066012854997576, + 0.0 + ], + [ + -0.0003700996719447213, + 0.01788426991484473, + 0.0042306209819743664, + -0.01867112802326085, + -0.006060401771293145, + -0.05416048440645393, + 0.0 + ], + [ + 0.0024717447389075923, + 0.017169032943787865, + -0.008648823801234667, + 0.00206897478697438, + 0.015393506079659652, + -0.04519078545059638, + 0.0 + ], + [ + 0.004519391134754436, + 0.015448564694053387, + -0.0008319571696841307, + -0.0054893642803115995, + 0.008801909341956949, + -0.01245290165551986, + 0.0 + ], + [ + 0.002921769198057308, + 0.01858911510664388, + 0.0025891746374897082, + -0.009075031174435411, + 0.004536043659506241, + -0.025237651605801263, + 0.0 + ], + [ + 0.002533945169148104, + 0.015392828874220985, + -0.002170755858424103, + 0.01330109124066637, + 0.016289059292800948, + -0.015027480559818463, + 0.0 + ], + [ + -0.003205796033002281, + 0.01346583529231473, + 0.011743497994674657, + -0.0010508245950151213, + -0.008319879712048622, + -0.02432804921873056, + 0.0 + ], + [ + -0.0019205718646689322, + 0.009006557471471374, + 0.0038862367033016525, + 0.00882704654341015, + 0.017873451118128405, + -0.03354251215567052, + 0.0 + ], + [ + -0.004040919073486391, + 0.0028121196163314326, + 0.0035155612365857817, + 0.011530223480562478, + 0.01880803388450074, + -0.024314569506220823, + 0.0 + ] + ], + "state": [ + [ + 0.4237840473651886, + 0.15348464250564575, + 0.06656582653522491, + -3.021669002371379, + -0.18417893350124362, + -0.6462129354476929, + 1.0 + ], + [ + 0.42310696840286255, + 0.15321387350559235, + 0.059661027044057846, + -3.034999558823653, + -0.17622175812721252, + -0.6438236832618713, + 1.0 + ], + [ + 0.4187030792236328, + 0.14651824533939362, + 0.03667914494872093, + -3.0894893800192555, + -0.18448191881179812, + -0.6526956558227539, + 1.0 + ], + [ + 0.4187246561050415, + 0.14683569967746735, + 0.031225282698869705, + -3.1033000034564218, + -0.2267358750104904, + -0.6827179789543152, + 1.0 + ], + [ + 0.4157203435897827, + 0.1486215740442276, + 0.022143902257084846, + -3.14033526415518, + -0.231326624751091, + -0.7001624107360841, + 1.0 + ], + [ + 0.4174812138080597, + 0.14965800940990448, + 0.021460002288222313, + 3.1408166628731458, + -0.24094812572002414, + -0.715813934803009, + 1.0 + ], + [ + 0.41862353682518005, + 0.15019799768924713, + 0.01757434383034706, + 3.1378910442632666, + -0.2486598193645477, + -0.7168003916740417, + 1.0 + ], + [ + 0.4204314649105072, + 0.15180018544197083, + 0.02050655521452427, + -3.1404754893207034, + -0.25024741888046265, + -0.7192215919494629, + 1.0 + ], + [ + 0.42066019773483276, + 0.15223529934883118, + 0.021163221448659897, + -3.1403925566752484, + -0.24765785038471222, + -0.7233408689498901, + 0.0 + ], + [ + 0.4203694760799408, + 0.1534765064716339, + 0.020762091502547264, + -3.133620496029504, + -0.23699024319648745, + -0.7251115441322326, + 0.0 + ], + [ + 0.41951414942741394, + 0.1504393070936203, + 0.021410487592220306, + -3.135631227987357, + -0.1932450979948044, + -0.7161328196525574, + 0.0 + ], + [ + 0.4157024025917053, + 0.14105801284313202, + 0.023109450936317444, + 3.1256677714460572, + -0.12157872319221495, + -0.6909732222557068, + 0.0 + ], + [ + 0.41706690192222595, + 0.13305220007896423, + 0.04000541940331459, + -3.1266413248428186, + -0.09844823926687242, + -0.6681079268455505, + 0.0 + ], + [ + 0.4163457155227661, + 0.12556125223636627, + 0.06110164895653725, + -3.0746804421120366, + -0.08374102413654327, + -0.6529660820960997, + 0.0 + ], + [ + 0.40865573287010193, + 0.11532784253358841, + 0.06082811579108238, + -3.0814870839291295, + -0.05255172774195671, + -0.6125687360763549, + 0.0 + ], + [ + 0.40160536766052246, + 0.10299066454172134, + 0.07759375125169754, + -3.059844100969382, + -0.08105576783418657, + -0.5519344210624696, + 0.0 + ], + [ + 0.39251774549484253, + 0.0880594328045845, + 0.07190554589033127, + -3.083057602989026, + -0.07931552827358246, + -0.49728840589523315, + 0.0 + ], + [ + 0.3857342600822449, + 0.07166403532028198, + 0.07970699667930603, + -3.0845576134794435, + -0.09724777191877364, + -0.45287182927131653, + 0.0 + ], + [ + 0.3830139636993408, + 0.05578247830271721, + 0.08009601384401321, + -3.091213861601897, + -0.10673826932907104, + -0.4408731460571289, + 0.0 + ], + [ + 0.37811383605003357, + 0.0377102866768837, + 0.07690533995628357, + -3.1029817295544824, + -0.11250589042901993, + -0.41573733091354365, + 0.0 + ], + [ + 0.3740096986293793, + 0.02261698618531227, + 0.07875481992959976, + -3.091308331238576, + -0.12935115396976474, + -0.40123066306114197, + 0.0 + ], + [ + 0.367534339427948, + 0.011396260000765324, + 0.06603945046663284, + -3.0955844802134713, + -0.12222477793693544, + -0.37633010745048523, + 0.0 + ], + [ + 0.3630078434944153, + 0.00370313273742795, + 0.06154099106788635, + -3.0907992516928395, + -0.14155596494674685, + -0.3433206081390381, + 0.0 + ], + [ + 0.3588409125804901, + 0.0023997207172214985, + 0.05735362693667412, + -3.082608251022645, + -0.16153523325920102, + -0.3196891248226165, + 0.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.06803375482559204, + 0.2779330015182495, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4850575923919678, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692, + 0.4840872883796692 + ], + "episode_id": "13109", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/13109/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/1432.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/1432.json new file mode 100644 index 00000000..0baa726d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/1432.json @@ -0,0 +1,888 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Move the pot in front of the yellow cloth" + ], + "videos": [ + { + "video_path": "videos/train/1432/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + 3.3653181170512016e-09, + 8.174184521982179e-10, + -0.003067941479543543, + 1.0 + ], + [ + -0.002871671105483313, + 0.005620327601906185, + 0.008487026639364112, + -0.028069226783864108, + -0.03184995475499298, + 0.005887884676629873, + 1.0 + ], + [ + 0.020371664859338137, + 0.025721468770555558, + 0.008223385257331559, + -0.08976916568744936, + -0.008897582996847914, + 0.07664856540501347, + 1.0 + ], + [ + 0.01800175380739509, + 0.017987981788555152, + 0.0013815202297006966, + -0.05151957501700951, + 0.03122935575346231, + 0.03662470493234054, + 1.0 + ], + [ + 0.02375799611305716, + 0.023129125292642088, + -0.0064171156404242155, + -0.05410797980071999, + 0.07223560916477792, + 0.04705595375613376, + 1.0 + ], + [ + 0.01628076829508081, + 0.017338720724991073, + 0.006213654056900202, + -0.08986360916648259, + 0.04658343561393271, + 0.01353182140474375, + 1.0 + ], + [ + 0.011723436207139388, + 0.013345065549968699, + 0.0007345802148524071, + -0.09409196831553919, + 0.05569585330556735, + -0.01326507636411717, + 1.0 + ], + [ + 0.0066028728150693584, + 0.004222510898741944, + 0.0114811500465903, + -0.08651933252361263, + 0.022946014109542032, + -0.036194885351024886, + 1.0 + ], + [ + 0.0016132492209657648, + -7.534892196091851e-05, + 0.01217428822171538, + -0.01011309242427128, + -0.018079466176755942, + -0.008678746702500963, + 1.0 + ], + [ + 0.007201347338746159, + 0.0089325748803942, + 0.016716102121627413, + 0.03679150143214499, + -0.025269116043175232, + 0.021644462773396014, + 1.0 + ], + [ + -0.00044938331541010524, + 0.0007849466602584071, + 0.013173614862204823, + 0.03086118956066966, + -0.017859752614394597, + 0.00024634080199766934, + 1.0 + ], + [ + 0.002746493185546987, + 0.002841002719481851, + 0.006777222993001901, + 0.04469814903773652, + -0.017051826126225415, + 0.017434003154150914, + 1.0 + ], + [ + 0.0006783896495208781, + -0.0019580356511931466, + 0.000677082447384172, + 0.04300293371142925, + -0.0016984967500821347, + 0.008445912851892525, + 1.0 + ], + [ + -0.0032582833032183452, + -0.0005919713777738123, + 0.0009288616422306345, + 0.021301139509369207, + -0.0247581101709915, + 1.4219816966240965e-05, + 0.0 + ], + [ + -0.0025662549718557365, + -0.00164878207987541, + 0.0004107645546126856, + 0.008600602511830873, + -0.00824087530677144, + 0.004661430223201045, + 0.0 + ], + [ + -0.0034847119835639258, + 0.0009203400024660716, + -0.006273950318016986, + 0.0001259368117794632, + -0.020312216741385777, + -0.0001120270968154216, + 0.0 + ], + [ + -0.0011226450225133755, + -0.0027171625122139535, + -0.012578016689818257, + 0.020224005147070543, + 0.013870593949600978, + 0.00029896553608349996, + 0.0 + ], + [ + -0.004238583822200444, + -0.005672446930486044, + -0.012459330110403592, + 0.036221434262490575, + 0.0012650814189358477, + -0.022471165181119298, + 0.0 + ], + [ + -0.005264820767006021, + -0.01684906270707055, + -0.025951418958079484, + 0.07587846700083131, + 0.010854505089523523, + -0.10834694194400896, + 0.0 + ], + [ + -0.006910853847470231, + -0.024212570333862513, + -0.016351441702158315, + 0.05462929345581949, + 8.936310902290299e-06, + -0.15102260057461903, + 0.0 + ], + [ + -0.004473925109092214, + -0.020326501055480582, + 0.00020510147858192497, + 0.00887860980355774, + -0.018891035091638756, + -0.09468523652213713, + 0.0 + ], + [ + 0.009076986311934408, + -0.019546329444008487, + -0.004266657084574958, + 0.01718660993625522, + 0.026224103288264022, + -0.06399261641648639, + 0.0 + ], + [ + 0.009206557957456807, + -0.022761463305509412, + 0.011179072163948471, + 0.023258442800202162, + -0.014424517192023374, + -0.04760804375431499, + 0.0 + ], + [ + 0.014095131211855644, + -0.02071961899939461, + 0.012334713854636815, + 0.01567338096199293, + -0.01050769123089415, + -0.04366939805142056, + 0.0 + ], + [ + 0.007510319140216454, + -0.011449166198868689, + 0.01295660845870957, + 0.016813167540613766, + -0.02343484680557583, + 0.010938353071650994, + 0.0 + ], + [ + 0.00498575845868187, + -0.004168869771974965, + 0.014277186676546736, + 0.01455062996495561, + -0.021122422767198005, + 0.03955242242087068, + 0.0 + ], + [ + 0.0006587013071789935, + -0.0054085287476437266, + 0.010880904374632653, + 0.01667857588575539, + -0.01158102986695254, + 0.049943244419845295, + 0.0 + ], + [ + 0.002374493850346168, + -0.004129919299659279, + 0.006865244244119527, + 0.017829632913467057, + 0.0021542298602684885, + 0.031098961211708263, + 0.0 + ], + [ + -0.0009224244320679821, + -0.0019296911600213313, + 0.0002826935038503009, + 0.012758463848807846, + -0.004226838644542556, + -0.012386288433056225, + 0.0 + ], + [ + -0.006978617139813965, + -0.0031055337471323535, + 0.0033806874576149904, + 0.01647385858505655, + -0.04419749927070117, + -0.02705002366447944, + 0.0 + ], + [ + 0.0004983531096983774, + 0.0016372151526067042, + 0.0004707417567702324, + -0.006350210943011992, + 0.0021280297380284275, + 0.0017935822423917809, + 1.0 + ], + [ + 0.0005875738354129934, + 0.00044164733789468113, + -0.00038912749570092, + 0.00015661570867947643, + 0.0017461789704789669, + -0.0017901335322080079, + 1.0 + ], + [ + -0.0007188451346825227, + -0.000258005103849561, + 0.000301487919067119, + 0.002120267131035959, + -0.0031640543834729113, + 0.009723672917114031, + 1.0 + ], + [ + 0.0006632889725949069, + -0.0002757358304051514, + -0.0022586814455474833, + -0.00012738950680019667, + 0.00604523682706346, + 0.011950353355767508, + 1.0 + ], + [ + 0.002125620484431426, + -0.0038092111742518827, + -0.021263585030410686, + 0.009736219092364752, + 0.047502975054863235, + 0.007558986432881434, + 1.0 + ], + [ + -0.0002445164832963776, + -0.004215956971539285, + -0.024724133145014662, + 0.012872732916890471, + 0.04847511081610798, + 0.015197812380810914, + 1.0 + ], + [ + -0.0007466831339435543, + -0.00022897060207310388, + -0.02363416026412954, + -0.008494962633356737, + 0.05439593358132928, + 0.01199513001417962, + 1.0 + ], + [ + -0.0008888798849157538, + -0.00015551413752249403, + 0.00013621769860253837, + -0.0012944685133784504, + -0.0015773753693677985, + 0.0009010874665147182, + 1.0 + ], + [ + -0.00010403244241662353, + -7.449407361842957e-05, + -0.0005458494386643956, + 0.0001431050708870694, + -4.22304202166489e-05, + -0.00020535815168722807, + 1.0 + ], + [ + 0.000104040193616245, + 7.459354838824717e-05, + 0.0005458343763641277, + -0.00014309639563635031, + 4.2259806686352804e-05, + 0.0002053521063806014, + 1.0 + ], + [ + -0.0002406909308538358, + 3.530870286486693e-05, + -1.863531243501104e-07, + -0.00022275432625192275, + -0.001517725207338478, + 1.7061589685801498e-07, + 1.0 + ], + [ + 0.0002406909304478395, + -3.530870319275974e-05, + -1.868148047877837e-07, + 0.0002227543238600442, + 0.0015177252076895322, + 1.674640236641407e-07, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 6.758109378461515e-18, + -1.2738159391686094e-17, + 1.2522983926599909e-17, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 6.758109378461515e-18, + -1.2738159391686094e-17, + 1.2522983926599909e-17, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 6.758109378461515e-18, + -1.2738159391686094e-17, + 1.2522983926599909e-17, + 1.0 + ] + ], + "state": [ + [ + 0.24887174367904663, + 0.13085252046585083, + 0.076690673828125, + -2.934714304404803, + -0.11415936052799225, + -0.025790268555283543, + 1.0 + ], + [ + 0.24887174367904663, + 0.13085252046585083, + 0.076690673828125, + -2.9350595345073422, + -0.11478901654481888, + -0.02276785485446453, + 1.0 + ], + [ + 0.2470172494649887, + 0.127133309841156, + 0.06696480512619019, + -2.963111104565211, + -0.08240872621536253, + -0.021994728595018387, + 1.0 + ], + [ + 0.26783207058906555, + 0.10281672328710556, + 0.05602544546127319, + -3.0472797473245343, + -0.05983198806643486, + -0.09596894681453703, + 1.0 + ], + [ + 0.2841981053352356, + 0.08338113129138947, + 0.05403801053762436, + -3.0964621288054666, + -0.08742783963680267, + -0.1355067193508148, + 1.0 + ], + [ + 0.3040235638618469, + 0.057066552340984344, + 0.06145908311009407, + 3.137118413700261, + -0.1573563814163208, + -0.18627980351448062, + 1.0 + ], + [ + 0.3175536096096039, + 0.036844540387392044, + 0.057950202375650406, + 3.049382323520728, + -0.20398570597171783, + -0.199870228767395, + 1.0 + ], + [ + 0.32605496048927307, + 0.02149508148431778, + 0.06081210449337959, + 2.951319607096263, + -0.25818815827369684, + -0.18092760443687442, + 1.0 + ], + [ + 0.33382895588874817, + 0.013650464825332165, + 0.052369870245456696, + 2.8543294091993054, + -0.27366575598716736, + -0.1395110785961151, + 1.0 + ], + [ + 0.3380279541015625, + 0.009650630876421928, + 0.04154437780380249, + 2.843276025848933, + -0.25386685132980347, + -0.13620562851428988, + 1.0 + ], + [ + 0.34642985463142395, + -0.005077731795608997, + 0.03042854368686676, + 2.887352170544215, + -0.23597240447998044, + -0.16511711478233337, + 1.0 + ], + [ + 0.3482241630554199, + -0.00950590055435896, + 0.01811862923204899, + 2.9193078150325498, + -0.2187460958957672, + -0.16996265947818753, + 1.0 + ], + [ + 0.35142502188682556, + -0.014382784254848957, + 0.012873080559074879, + 2.9686246534162244, + -0.2059092074632645, + -0.19117304682731626, + 1.0 + ], + [ + 0.3526228666305542, + -0.012768694199621677, + 0.012029020115733147, + 3.0134325345330915, + -0.20568189024925235, + -0.19997069239616394, + 1.0 + ], + [ + 0.34978941082954407, + -0.011716487817466259, + 0.010387674905359745, + 3.0353548919134816, + -0.1811276376247406, + -0.20320165157318118, + 0.0 + ], + [ + 0.3477420508861542, + -0.00966518372297287, + 0.00935165211558342, + 3.0449609850817403, + -0.17342473566532135, + -0.20879438519477844, + 0.0 + ], + [ + 0.3432506024837494, + -0.009031150490045547, + 0.014988813549280167, + 3.0453898405008992, + -0.15319618582725522, + -0.21066442131996155, + 0.0 + ], + [ + 0.3411545157432556, + -0.0045816972851753235, + 0.026932790875434875, + 3.0654444714360913, + -0.1670312732458115, + -0.20961508154869082, + 0.0 + ], + [ + 0.3364905118942261, + 0.003162156790494919, + 0.03805290535092354, + 3.0978918505185327, + -0.16654066741466522, + -0.1867970377206802, + 0.0 + ], + [ + 0.33062225580215454, + 0.02255605347454548, + 0.062022171914577484, + -3.1274152576276144, + -0.1716674864292145, + -0.0764549970626831, + 0.0 + ], + [ + 0.32282108068466187, + 0.04720223322510719, + 0.07728931307792664, + -3.099032415198632, + -0.17183594405651093, + 0.07682709395885468, + 0.0 + ], + [ + 0.31675419211387634, + 0.06711214035749435, + 0.07717451453208923, + -3.106829261528798, + -0.15619739890098572, + 0.17339599132537845, + 0.0 + ], + [ + 0.3214840590953827, + 0.08762195706367493, + 0.08346981555223465, + -3.099669685708829, + -0.18431410193443296, + 0.23750194907188416, + 0.0 + ], + [ + 0.3266395032405853, + 0.11275027692317963, + 0.07511484622955322, + -3.085412172722169, + -0.1716810166835785, + 0.2863876223564148, + 0.0 + ], + [ + 0.33574986457824707, + 0.13772040605545044, + 0.06653495877981186, + -3.0774395634704312, + -0.16347214579582214, + 0.33117371797561646, + 0.0 + ], + [ + 0.34064921736717224, + 0.15236549079418182, + 0.05572386085987091, + -3.059064658480235, + -0.1393769979476929, + 0.3216710388660431, + 0.0 + ], + [ + 0.345477819442749, + 0.15959446132183075, + 0.042666222900152206, + -3.039286309974738, + -0.11496751010417938, + 0.28375405073165894, + 0.0 + ], + [ + 0.34541982412338257, + 0.1663394421339035, + 0.03253794461488724, + -3.017133029299327, + -0.09821285307407379, + 0.23502463102340698, + 0.0 + ], + [ + 0.34716588258743286, + 0.17184768617153168, + 0.026501674205064774, + -2.9962978978925427, + -0.09644275158643724, + 0.2037534564733505, + 0.0 + ], + [ + 0.3458721339702606, + 0.1735718548297882, + 0.02641252614557743, + -2.984794782596179, + -0.0940457284450531, + 0.216678261756897, + 0.0 + ], + [ + 0.3385756313800812, + 0.1756470799446106, + 0.02291553094983101, + -2.971417160826274, + -0.05456070229411125, + 0.25032296776771545, + 0.0 + ], + [ + 0.3394768536090851, + 0.17429432272911072, + 0.022202597931027412, + -2.977651136117526, + -0.05635411664843559, + 0.24819152057170868, + 1.0 + ], + [ + 0.3401511311531067, + 0.1739501804113388, + 0.022547027096152306, + -2.9775780161195478, + -0.0583689846098423, + 0.24967518448829648, + 1.0 + ], + [ + 0.339395135641098, + 0.17407089471817017, + 0.022250212728977203, + -2.9749334921413144, + -0.053657323122024536, + 0.24058562517166138, + 1.0 + ], + [ + 0.3399444818496704, + 0.1740998476743698, + 0.024555642157793045, + -2.9743846227699002, + -0.05763203650712967, + 0.227776899933815, + 1.0 + ], + [ + 0.34075063467025757, + 0.174509659409523, + 0.04624320939183235, + -2.9635880311303815, + -0.10320723056793212, + 0.212346151471138, + 1.0 + ], + [ + 0.3380352258682251, + 0.17369051277637482, + 0.07116453349590302, + -2.9481012244992932, + -0.1481963098049164, + 0.18856379389762878, + 1.0 + ], + [ + 0.334749311208725, + 0.16866567730903625, + 0.09403672814369202, + -2.9530047943168363, + -0.19923280179500583, + 0.16590565443038943, + 1.0 + ], + [ + 0.3338809609413147, + 0.16870105266571045, + 0.093758225440979, + -2.9541800936036786, + -0.19751442968845365, + 0.16530457139015198, + 1.0 + ], + [ + 0.33367854356765747, + 0.16863837838172913, + 0.0942772850394249, + -2.9540789445214948, + -0.19751119613647464, + 0.1655183583498001, + 1.0 + ], + [ + 0.3338809609413147, + 0.16870105266571045, + 0.093758225440979, + -2.9541800936036786, + -0.19751442968845365, + 0.16530457139015198, + 1.0 + ], + [ + 0.33365511894226074, + 0.16862817108631134, + 0.0937047228217125, + -2.954459177451678, + -0.19602324068546295, + 0.16559270024299622, + 1.0 + ], + [ + 0.3338809609413147, + 0.16870105266571045, + 0.093758225440979, + -2.9541800936036786, + -0.19751442968845365, + 0.16530457139015198, + 1.0 + ], + [ + 0.3338809609413147, + 0.16870105266571045, + 0.093758225440979, + -2.9541800936036786, + -0.19751442968845365, + 0.16530457139015198, + 1.0 + ], + [ + 0.3338809609413147, + 0.16870105266571045, + 0.093758225440979, + -2.9541800936036786, + -0.19751442968845365, + 0.16530457139015198, + 1.0 + ], + [ + 0.3338809609413147, + 0.16870105266571045, + 0.093758225440979, + -2.9541800936036786, + -0.19751442968845365, + 0.16530457139015198, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.05698961019515991, + 0.34891170263290405, + 0.6497291326522827, + 0.8586209267377853, + 0.9207848757505417, + 0.9221144318580627, + 0.9221144318580627, + 0.9224447533488274, + 0.9224447533488274, + 0.9221144318580627, + 0.9224447533488274, + 0.9224447533488274, + 0.9224447533488274, + 0.9224447533488274, + 0.9224447533488274, + 0.9224447533488274, + 0.9224447533488274, + 0.8888042271137238, + 0.7079049050807953, + 0.42306244373321533, + 0.12860965728759766, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "1432", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/1432/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/19139.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/19139.json new file mode 100644 index 00000000..3ecc6272 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/19139.json @@ -0,0 +1,717 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Move blue cloth to front of cans." + ], + "videos": [ + { + "video_path": "videos/train/19139/rgb.mp4" + } + ], + "action": [ + [ + 0.00013624903765218385, + 4.671830144473438e-06, + 0.0005918521112988113, + -1.3323561015808925e-05, + 9.599008361969608e-07, + -0.0015064613707889247, + 1.0 + ], + [ + -0.006673673911223424, + 0.00047502714766228175, + 0.015976769330000928, + -0.0004891007491801375, + -0.0383456577357582, + -0.012148261678623219, + 1.0 + ], + [ + -0.019930288929694326, + 0.012639964346488914, + 0.02099136413189724, + 0.010867030165416505, + -0.007849399477978473, + -0.0371325378439439, + 1.0 + ], + [ + -0.016151195835974214, + 0.010701087616745633, + -0.016300094969435627, + -0.029484421660474464, + 0.0595453553086942, + 0.026093279263275836, + 1.0 + ], + [ + -0.029609583776693073, + 0.016539731741262845, + -0.0016461004493825329, + -0.0301977895238755, + -0.003464926669261775, + 0.03267068901736831, + 1.0 + ], + [ + -0.02822718494242145, + 0.0201980028605076, + 0.007771125192385183, + -0.018168714064400654, + -0.026044919700338565, + 0.04587069237592233, + 1.0 + ], + [ + -0.0194968579827593, + 0.015172011461582641, + 0.011524601063444892, + -0.0006086171815272609, + -0.02762203433878553, + 0.04977419220928906, + 1.0 + ], + [ + -0.006979029022660864, + 0.008964887940227673, + 0.005065222903008267, + 0.024000831860584222, + -0.006958677884872038, + 0.033785727526092345, + 1.0 + ], + [ + -0.0010456977633956986, + 0.0002466314084411563, + 0.006346013264108883, + 0.02710641002994572, + -0.010719191672451804, + 0.005876491130813615, + 1.0 + ], + [ + -0.001746870710019766, + 0.00065671325189339, + 0.006701352648434884, + 0.01834056030256817, + -0.015364078976297886, + -0.00013077410611913763, + 1.0 + ], + [ + 0.0005307283083640442, + -0.0017744763150722193, + -0.0005688547189640877, + -0.001656987943727148, + 0.0008608498178647336, + -0.0023587448838887208, + 1.0 + ], + [ + 0.00704620461843825, + -0.012227625026128296, + -0.010580654672332311, + -0.03248876490306802, + 0.05111471962511007, + -0.06826083071951623, + 1.0 + ], + [ + 0.0022464023041225472, + -0.007327294935972523, + 0.0011282471567426984, + -0.024380302950346395, + 0.004708733369595492, + -0.02841747253279181, + 1.0 + ], + [ + 0.0020652279823257214, + -0.0008988600916203636, + 0.00806662511477008, + 0.0003942348917254095, + -0.018390486125310294, + -0.01783192515265195, + 1.0 + ], + [ + 0.0002834925636690119, + 0.0016274863909312408, + 0.0136358626398185, + 0.012967831840491823, + -0.028131365896626407, + 0.004521698821186212, + 1.0 + ], + [ + 0.005287764949264742, + 0.0017076963079391088, + 0.006753554152672276, + 0.00716964960841718, + -0.006923507779503204, + 0.00812243741561222, + 1.0 + ], + [ + 0.0006046067484475125, + 0.000537725577220217, + 0.004202703634653392, + 0.022467617515206, + -0.004820950339894369, + 0.017740516943045098, + 1.0 + ], + [ + 0.0005362944678515512, + -0.0003966269253268623, + 0.00044330626608841224, + 0.010406973156943966, + 0.00790921804097952, + 0.0003985580854949119, + 1.0 + ], + [ + 0.013879684283115637, + 0.004319351931534076, + -0.00013432181212480481, + -0.0003963268871553246, + 0.02216787623314653, + -0.005175642062883274, + 1.0 + ], + [ + 0.02524390624910577, + 0.006045279885585814, + 0.002555644409063367, + -0.02748794370478586, + -0.0010158942195905541, + -0.04413847303886072, + 1.0 + ], + [ + 0.03804403306660603, + 0.0022059895510740516, + 0.003585913654107192, + -0.023427087697595453, + -0.003269637214120781, + -0.046154997425420084, + 1.0 + ], + [ + 0.035650788100130344, + 0.004532142424352461, + 0.0031678763638340593, + 0.00834956295496929, + 0.0059948128043535885, + 0.02214228118420332, + 1.0 + ], + [ + 0.024142304764812925, + 0.0033009639157718837, + 0.002457195437731301, + 0.0040276537447393335, + -0.0010417315849879645, + 0.037774216092047605, + 1.0 + ], + [ + 0.005659693931113641, + 0.002782422008804238, + -0.0006595606193741893, + 0.0030234435403949803, + 0.0034247317924557185, + 0.029915878179651408, + 1.0 + ], + [ + -0.0030420336017515075, + 0.0003436255418653344, + -0.002004323781668532, + 0.016344181309362084, + -0.0013129054295395403, + 0.013592632928259208, + 1.0 + ], + [ + 0.0026904147997999974, + 0.0036451900899932468, + -0.007545917642150608, + -0.02037534475136874, + 0.01424693896522226, + -0.03323627487970372, + 1.0 + ], + [ + 0.014807428909307746, + 0.011185528684004718, + -0.022056505672639573, + -0.05021397761601606, + 0.024461878448517962, + -0.00502564846541712, + 1.0 + ], + [ + 0.008790868210456056, + 0.005975934656390852, + -0.032339954498121555, + -0.02772411916873434, + 0.04312300939783141, + 0.004472967244709832, + 1.0 + ], + [ + -3.1985978810743963e-05, + 0.002804965216245947, + -0.032153077412060055, + -0.009101036972034796, + 0.032902573845877615, + 0.017591473211817903, + 1.0 + ], + [ + -0.019394442644223214, + -0.0016942604345596389, + -0.03984419678082064, + 0.004567622097686483, + 0.03743782204220558, + -0.000526895622745784, + 1.0 + ], + [ + -0.03407785758807691, + -0.012608211425703902, + -0.026579019547746833, + 0.028083353132591762, + 0.016063636181958486, + -0.011634098661703073, + 1.0 + ], + [ + -0.0440360162280434, + -0.016842445740729524, + -0.004151387569918484, + 0.04092458867415413, + -0.01254447434233229, + 0.0015934580902097397, + 1.0 + ], + [ + -0.016787827737485102, + -0.015011209407362723, + 0.014814138610554814, + 0.02053646052485796, + -0.030834116146731892, + -0.002431953858922028, + 1.0 + ], + [ + -0.0006719023786519985, + -0.011832750186327295, + 0.019264938293728853, + 0.014712967464073742, + -0.03198998368377831, + 0.0025188750637281797, + 1.0 + ], + [ + 3.772070448571133e-05, + -0.004858856636255666, + 0.00824142223972118, + 0.002923165090997738, + -0.012489335856414189, + 0.0053227395296207335, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + -2.832438230221676e-18, + -3.996656921287466e-18, + -1.9046282367766035e-18, + 1.0 + ] + ], + "state": [ + [ + 0.39123794436454773, + 0.02086617983877659, + 0.09526966512203217, + -3.115345500531145, + -0.1383592039346695, + -0.013857951387763023, + 1.0 + ], + [ + 0.39145463705062866, + 0.020874040201306343, + 0.0947023406624794, + -3.1155685522877654, + -0.1383995413780212, + -0.012337495572865008, + 1.0 + ], + [ + 0.38704946637153625, + 0.02086925506591797, + 0.07795076817274094, + -3.117862967895814, + -0.10037101060152052, + 0.0008617936400696635, + 1.0 + ], + [ + 0.36936280131340027, + 0.0087156742811203, + 0.05477546155452728, + -3.110765582072087, + -0.09333464503288269, + 0.038332268595695496, + 1.0 + ], + [ + 0.352236270904541, + -0.003147786483168602, + 0.06916322559118271, + -3.137575373640754, + -0.15201085805892944, + 0.010137512348592281, + 1.0 + ], + [ + 0.32289817929267883, + -0.019992273300886154, + 0.06624098867177963, + 3.1204091125452003, + -0.1483331024646759, + -0.02288150787353516, + 1.0 + ], + [ + 0.29560720920562744, + -0.039731115102767944, + 0.054808370769023895, + 3.109165074425288, + -0.12310499697923662, + -0.06963260471820833, + 1.0 + ], + [ + 0.2765744626522064, + -0.05397909879684448, + 0.041471078991889954, + 3.114831030490347, + -0.09695243835449219, + -0.12049726396799086, + 1.0 + ], + [ + 0.2690485715866089, + -0.06223149597644806, + 0.03599463403224945, + -3.1410383521048395, + -0.09084408730268478, + -0.15459704399108884, + 1.0 + ], + [ + 0.2685510218143463, + -0.062400005757808685, + 0.02957978844642639, + -3.1133976375036916, + -0.08012007176876067, + -0.16048614680767057, + 1.0 + ], + [ + 0.26728811860084534, + -0.06266918778419495, + 0.022744324058294296, + -3.095098978030034, + -0.0647657737135887, + -0.15992113947868344, + 1.0 + ], + [ + 0.2680474519729614, + -0.06102297827601433, + 0.023428020998835564, + -3.0969063026183328, + -0.06573514640331268, + -0.15759994089603424, + 1.0 + ], + [ + 0.2761131525039673, + -0.050414640456438065, + 0.034983180463314056, + -3.1337915236414453, + -0.11970251798629762, + -0.09130901098251343, + 1.0 + ], + [ + 0.2791304588317871, + -0.04332435503602028, + 0.03418805077672005, + 3.1215953679108104, + -0.12458430230617523, + -0.06270772963762283, + 1.0 + ], + [ + 0.28222429752349854, + -0.04277978464961052, + 0.026424342766404152, + 3.1198074744367084, + -0.10582190006971358, + -0.04515194147825241, + 1.0 + ], + [ + 0.28385379910469055, + -0.04477950930595398, + 0.012933173216879368, + 3.133310843380638, + -0.07779431343078613, + -0.050299070775508874, + 1.0 + ], + [ + 0.28955331444740295, + -0.04683223366737366, + 0.0066253188997507095, + 3.1411178438335234, + -0.0709357038140297, + -0.05849899724125862, + 1.0 + ], + [ + 0.29042115807533264, + -0.04742370545864105, + 0.0024762919638305902, + -3.1183395461463412, + -0.0661119967699051, + -0.07628044486045839, + 1.0 + ], + [ + 0.29101431369781494, + -0.047061026096343994, + 0.002078705234453082, + -3.107893271493264, + -0.07400979846715927, + -0.07686437666416168, + 1.0 + ], + [ + 0.30448412895202637, + -0.05243264138698578, + 0.0030937448609620333, + -3.10861041222746, + -0.09633874148130418, + -0.07241918146610261, + 1.0 + ], + [ + 0.329378217458725, + -0.06021198257803917, + 0.00278111407533288, + -3.1403956237533173, + -0.09668428450822833, + -0.028063932433724407, + 1.0 + ], + [ + 0.36751416325569153, + -0.06348507106304169, + 0.002881854074075818, + 3.1148875207924327, + -0.0933666080236435, + 0.018296770751476284, + 1.0 + ], + [ + 0.4033720791339874, + -0.0674448013305664, + 0.0031731491908431053, + 3.1253018510812005, + -0.0999278575181961, + -0.003787292167544365, + 1.0 + ], + [ + 0.42762085795402527, + -0.07087721675634384, + 0.0031905167270451784, + 3.1331285537504634, + -0.09942989796400072, + -0.04176110029220581, + 1.0 + ], + [ + 0.4330640137195587, + -0.07388382405042648, + 0.004432046320289373, + 3.1391377652813723, + -0.10306311398744582, + -0.07180620729923247, + 1.0 + ], + [ + 0.4298158586025238, + -0.07398976385593414, + 0.006113600917160511, + -3.126297155516692, + -0.10177402198314667, + -0.08547273278236389, + 1.0 + ], + [ + 0.4314034581184387, + -0.0778997465968132, + 0.013837471604347229, + 3.1331295195319733, + -0.11647181212902068, + -0.05223647505044937, + 1.0 + ], + [ + 0.4429456889629364, + -0.08951670676469803, + 0.037558529525995255, + 3.0822989774220666, + -0.14088867604732513, + -0.046953156590461724, + 1.0 + ], + [ + 0.4468718469142914, + -0.09375465661287308, + 0.07110683619976044, + 3.0547922794991216, + -0.18420065939426422, + -0.048892270773649216, + 1.0 + ], + [ + 0.4409356117248535, + -0.09347120672464371, + 0.10283017158508301, + 3.0484062006049832, + -0.2184841185808182, + -0.06391290575265884, + 1.0 + ], + [ + 0.41383832693099976, + -0.08633146435022354, + 0.1372007429599762, + 3.0520106946402272, + -0.25570860505104065, + -0.05977114289999008, + 1.0 + ], + [ + 0.37542062997817993, + -0.06907026469707489, + 0.15310178697109222, + 3.0766714086108884, + -0.27064478397369385, + -0.046256061643362045, + 1.0 + ], + [ + 0.3330080211162567, + -0.05001245066523552, + 0.14426758885383606, + 3.118255818384238, + -0.2582293152809143, + -0.04874228313565254, + 1.0 + ], + [ + 0.32137733697891235, + -0.03476622328162193, + 0.12532277405261993, + 3.138332590116658, + -0.22734646499156952, + -0.04698635637760163, + 1.0 + ], + [ + 0.3256223201751709, + -0.023182939738035202, + 0.10636463761329651, + -3.129538694518157, + -0.1953640729188919, + -0.04965904355049133, + 1.0 + ], + [ + 0.32749179005622864, + -0.018311889842152596, + 0.09834535419940948, + -3.1255937722795686, + -0.18280884623527527, + -0.054918073117733, + 1.0 + ], + [ + 0.32749179005622864, + -0.018311889842152596, + 0.09834535419940948, + -3.1255937722795686, + -0.18280884623527527, + -0.054918073117733, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0005114078521728516, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926, + 0.0011776089668273926 + ], + "episode_id": "19139", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/19139/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/2277.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/2277.json new file mode 100644 index 00000000..8ca4f15d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/2277.json @@ -0,0 +1,945 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Take the fish piece and place it on the blue cloth." + ], + "videos": [ + { + "video_path": "videos/train/2277/rgb.mp4" + } + ], + "action": [ + [ + -0.0004219165862856827, + 0.000254168208014854, + -0.0005765892792793622, + -0.0017222414673772276, + 0.00012173496431256832, + 0.0013133682179283111, + 1.0 + ], + [ + -0.002374482832746035, + -0.006045339449127297, + 0.011715855894030168, + 0.03628480537867172, + -0.012940276616998406, + 0.001679722685351351, + 1.0 + ], + [ + 0.010540762724008257, + -0.024470704240238995, + 0.017059187884621994, + 0.052025692385461036, + -0.0060322418561737775, + -0.03630445568586694, + 1.0 + ], + [ + 0.012558908151675862, + -0.017510837165919765, + 0.0009783943991188858, + -0.019871285290187307, + 0.016295934134038657, + -0.04695012837361725, + 1.0 + ], + [ + 0.011535906101690389, + -0.01990538676972562, + 0.011443251243160616, + 0.027471480535784163, + -0.004113700692959027, + -0.022438346788961106, + 1.0 + ], + [ + 0.004676338653418472, + -0.017472553118882808, + 0.00981481117837635, + 0.03663907723366434, + 0.036443917170297574, + 0.043586771646955506, + 1.0 + ], + [ + 0.006572585730990188, + -0.008747437162609256, + 0.00904402387605131, + 0.011841038408788461, + 0.04511603714415104, + 0.021920251222817613, + 1.0 + ], + [ + 0.008366800511121462, + -0.004043917789832211, + 0.010185865705491959, + 0.01265849060481568, + 0.009956080255441634, + -0.008224084726838624, + 1.0 + ], + [ + 0.006748766503483464, + -0.007745648017179148, + 0.017712273861077356, + 0.037664044969623396, + -0.004165169742364359, + -0.008982251597299306, + 1.0 + ], + [ + 0.005471691440343436, + -0.005711364534418811, + 0.014957943964793429, + 0.029830898900644033, + -0.00408232302084337, + -0.008628779058537869, + 1.0 + ], + [ + 0.002289489987343083, + -0.004914605014315859, + 0.010397412588470004, + 0.016551411958012775, + -0.0034492845341825497, + -3.0669413436707484e-05, + 1.0 + ], + [ + -0.005175972322928676, + -0.002355546984321684, + 0.012576486761620436, + 0.018427971890277645, + -0.020004743729160952, + 0.019324707592114892, + 1.0 + ], + [ + -0.0023277850217423663, + 0.0021774528081883716, + 0.004232453903789928, + -0.004487792237798138, + -0.014530794799909546, + -0.00949315453848612, + 1.0 + ], + [ + -0.0009942898755817176, + 0.0003273091631956673, + 0.007758803157843195, + 0.010596242057098396, + -0.004078015239739286, + -0.00047059491710570124, + 1.0 + ], + [ + -0.0026558290112963685, + -0.004900066104599246, + 0.003717427321820484, + 0.012881272949343348, + 0.003032213288355432, + -0.004981611967012464, + 1.0 + ], + [ + -0.0011570166108713737, + 0.00010996048857753948, + 0.0004287458681574413, + -0.015657826412065128, + 0.0026343714052520987, + -0.0028594334755009713, + 0.0 + ], + [ + -0.0019334174614646718, + -0.001273911293311492, + -0.0007646520021831611, + 0.00028280953612107605, + -0.008390073041849336, + 0.001274209875573609, + 0.0 + ], + [ + -0.001063222657655898, + -0.000960411334138261, + -0.003387105022447763, + 0.002212126397081432, + -0.008681897446681047, + -0.00462130752314687, + 0.0 + ], + [ + 0.004142246850934011, + 0.004749317503944503, + -0.023917411472044188, + -0.04015856131473903, + -0.008530697257752906, + -0.04584130281148584, + 0.0 + ], + [ + 0.007697523427699104, + 0.010482282526979027, + -0.03191375070008, + -0.04917590649184765, + 0.006553837813727759, + -0.05703459791877952, + 0.0 + ], + [ + -0.00926341048949264, + 0.016459153148771232, + -0.03131342266149356, + -0.04701514756401812, + 0.008507909861994864, + 0.0044101763754059, + 0.0 + ], + [ + -0.02665726325604569, + 0.021096662676495043, + -0.030755441006437068, + -0.06499119572124724, + -0.03890529341032356, + 0.007730708228739821, + 0.0 + ], + [ + -0.03658781178685986, + 0.01838391184583117, + -0.01563144204995185, + -0.040905791851994416, + -0.04910160415084793, + 0.047355943610683106, + 0.0 + ], + [ + -0.027429884831553078, + 0.007369419335984045, + -0.0010829423597021222, + -0.01108794664851331, + -0.0237788452409342, + 0.03993245033112962, + 0.0 + ], + [ + -0.03124392393346085, + 0.010728482088966042, + 0.014028208194049346, + -0.0038034625505695154, + -0.012905485358633081, + 0.06747154308022803, + 0.0 + ], + [ + -0.02429094719437563, + 0.01958115144468657, + 0.008790146471091312, + -0.01933457039564906, + -0.01474267566587553, + 0.046780701610286884, + 0.0 + ], + [ + -0.02369430306025632, + 0.011664535764067915, + 0.014191245761859622, + 0.003321931911542727, + -0.0021082330212060284, + 0.05564717127516734, + 0.0 + ], + [ + -0.017292412424666795, + 0.010842119710826766, + 0.0153572607911446, + -0.0027612302020619973, + -0.013634397842106193, + 0.025966677702340745, + 0.0 + ], + [ + -0.013405018933029412, + 0.006692154535243639, + 0.01636795104757035, + 0.001822746807512587, + -0.011972904260231869, + 0.04575089048632341, + 0.0 + ], + [ + -0.006807933631203818, + -0.002488053824866136, + 0.019075701793794166, + 0.031616938481365944, + -0.009127602516226699, + 0.03151859696108362, + 0.0 + ], + [ + -0.0016152766764658323, + -0.007635285061634899, + 0.006161773238402084, + 0.01651877452081035, + -0.0026095286373022148, + 0.014368455900871458, + 0.0 + ], + [ + -0.0020319546585975123, + -0.01060883267922648, + 0.012091542675570327, + 0.03188874878790806, + -0.001251841566390941, + 0.011086842593546439, + 0.0 + ], + [ + -0.0015368958671413589, + -0.0016978651584461804, + 0.0055087795429140975, + 0.011576400089932447, + -0.006880410058756575, + 0.006349462184395389, + 0.0 + ], + [ + 0.0017736691281740812, + -0.0013361421659353237, + 0.0005083918171977924, + 0.008650853387004933, + 0.011443399759414302, + -0.003558674965462075, + 0.0 + ], + [ + 0.0015002465790986885, + -0.005481791924801231, + 0.0022244358670445922, + 0.037322588841260204, + 0.011311374968241356, + -0.025108485691436564, + 0.0 + ], + [ + 0.006653349834272537, + -0.003846305903077538, + -0.000761619606663063, + 0.026390990090500218, + 0.01533030588948224, + -0.02674417080303261, + 0.0 + ], + [ + 0.004052518066200824, + -0.0004912859338105934, + -0.0010648594246614237, + 0.008265937550334905, + 0.017483452378552165, + -0.005517088613123199, + 1.0 + ], + [ + 0.0013324972241405674, + -0.0009664539203302994, + -8.541994091315514e-06, + 0.005944246596996796, + 0.0085103109378048, + -0.017795855990900805, + 1.0 + ], + [ + 0.005346879703109662, + 0.00035187978339038266, + -0.015921455164046035, + -0.024214749307916436, + -0.010021919101490698, + -0.029905684655145696, + 1.0 + ], + [ + 0.015825571283668845, + -0.003723020636568434, + -0.029160198653871757, + -0.01510319872998846, + -0.06450841866751565, + -0.06045276656082365, + 1.0 + ], + [ + 0.00889561286531893, + -0.004686775353220252, + -0.03561244829956955, + -0.04394346436546597, + 0.010029056641651435, + -0.026660728691474907, + 1.0 + ], + [ + 0.018336967330821824, + -0.007843601302749362, + -0.02507502167750987, + -0.04033441405139121, + 0.044050156330335544, + -0.05107312833175928, + 1.0 + ], + [ + 0.015649493713062804, + -0.004752396076237847, + -0.01643365066554718, + -0.038058548402255594, + 0.031217008262446416, + -0.02977903756841232, + 1.0 + ], + [ + 0.0009734128832633189, + -0.00014494219525363835, + 0.00010981253665424575, + 0.00044247537966724047, + 0.0007510943438317447, + -0.0029413495993276416, + 1.0 + ], + [ + -7.888006911391653e-05, + -0.00041489723043175404, + 0.0003362927019193281, + 0.0015159197189589235, + -0.000129882454145689, + 0.00019536323740544862, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 4.064046616705326e-17, + -3.2927609937500916e-17, + -3.0704318096300303e-18, + 1.0 + ], + [ + 4.475810550869971e-06, + 0.0002431921019283746, + -1.8225831124959756e-07, + -0.0015337286319287353, + 2.8229833036571385e-05, + -2.408927082785681e-08, + 1.0 + ], + [ + -4.475809835887824e-06, + -0.00024319209534389162, + -1.9085879797819264e-07, + 0.0015337286318598917, + -2.822983678017391e-05, + -1.920764374371126e-08, + 1.0 + ] + ], + "state": [ + [ + 0.33099564909935, + 0.11170126497745514, + 0.15515448153018951, + 2.8988503386550626, + -0.1798458993434906, + 1.564736247062683, + 1.0 + ], + [ + 0.33110058307647705, + 0.11117446422576904, + 0.1556897759437561, + 2.897354768710681, + -0.18027961254119873, + 1.5634701251983643, + 1.0 + ], + [ + 0.3280675709247589, + 0.11116116493940353, + 0.14264193177223206, + 2.934487076597758, + -0.16812752187252045, + 1.5586432218551636, + 1.0 + ], + [ + 0.30779963731765747, + 0.12543630599975586, + 0.12298563122749329, + 2.980826350050517, + -0.15466083586215973, + 1.5933440923690796, + 1.0 + ], + [ + 0.2903827130794525, + 0.13803640007972717, + 0.1211964339017868, + 2.9534887244277677, + -0.16304725408554077, + 1.6429520845413208, + 1.0 + ], + [ + 0.27201786637306213, + 0.15055689215660095, + 0.10830383002758026, + 2.9775068928771695, + -0.15477335453033447, + 1.6644823551177979, + 1.0 + ], + [ + 0.2558387815952301, + 0.15561866760253906, + 0.09663745015859604, + 3.0199733843379697, + -0.19773826003074646, + 1.6267305612564087, + 1.0 + ], + [ + 0.247794508934021, + 0.16359801590442657, + 0.08808580785989761, + 3.035003202157565, + -0.24515327811241153, + 1.6099656820297241, + 1.0 + ], + [ + 0.24444125592708588, + 0.17415398359298706, + 0.07987388223409653, + 3.045344801741191, + -0.25416722893714905, + 1.6195085048675537, + 1.0 + ], + [ + 0.23789769411087036, + 0.18500028550624847, + 0.06378646194934845, + 3.0807957371049604, + -0.24914869666099548, + 1.6283209323883057, + 1.0 + ], + [ + 0.23259219527244568, + 0.1937795877456665, + 0.050330061465501785, + 3.10850244958932, + -0.24454075098037717, + 1.6369421482086182, + 1.0 + ], + [ + 0.2277185320854187, + 0.1982439011335373, + 0.04064406082034111, + 3.1250744742625436, + -0.2410923391580582, + 1.636856198310852, + 1.0 + ], + [ + 0.2257087081670761, + 0.19609194993972778, + 0.02715936489403248, + -3.1348736133710524, + -0.22136224806308746, + 1.6167160272598267, + 1.0 + ], + [ + 0.22791685163974762, + 0.19485348463058472, + 0.02250492013990879, + -3.1415125507243253, + -0.20688524842262268, + 1.6265145540237427, + 1.0 + ], + [ + 0.22820846736431122, + 0.19549138844013214, + 0.014707304537296295, + -3.131015065247663, + -0.2028072476387024, + 1.626995325088501, + 1.0 + ], + [ + 0.22338180243968964, + 0.1933537870645523, + 0.010582100600004196, + -3.119152257340499, + -0.20588946342468264, + 1.6320513486862183, + 1.0 + ], + [ + 0.2235458642244339, + 0.1923174411058426, + 0.009923562407493591, + -3.1353951186225792, + -0.2085865139961243, + 1.6349129676818848, + 0.0 + ], + [ + 0.22241076827049255, + 0.19018885493278503, + 0.010278983041644096, + -3.1348538511716804, + -0.200188547372818, + 1.633665919303894, + 0.0 + ], + [ + 0.22158291935920715, + 0.18841654062271118, + 0.013393285684287548, + -3.1335894550480923, + -0.19153577089309692, + 1.6384326219558716, + 0.0 + ], + [ + 0.226544588804245, + 0.18827234208583832, + 0.037623800337314606, + 3.1005441044741353, + -0.18316811323165894, + 1.685120701789856, + 0.0 + ], + [ + 0.2354566603899002, + 0.19098907709121704, + 0.07080228626728058, + 3.040826255577155, + -0.18707340955734253, + 1.7433931827545166, + 0.0 + ], + [ + 0.2510969638824463, + 0.17828238010406494, + 0.10131729394197464, + 2.994477780657359, + -0.19598062336444852, + 1.7397924661636353, + 0.0 + ], + [ + 0.2727198302745819, + 0.14882613718509674, + 0.1290004551410675, + 2.932020681100436, + -0.15861137211322784, + 1.726279616355896, + 0.0 + ], + [ + 0.29333412647247314, + 0.1124318316578865, + 0.1420956254005432, + 2.9001871814304074, + -0.12017662078142168, + 1.669381022453308, + 0.0 + ], + [ + 0.3029109835624695, + 0.0856751874089241, + 0.14160016179084778, + 2.8946313132816037, + -0.10651125758886336, + 1.6246753931045532, + 0.0 + ], + [ + 0.31833338737487793, + 0.05656389519572258, + 0.12736092507839203, + 2.8987288643890103, + -0.11022638529539108, + 1.555678129196167, + 0.0 + ], + [ + 0.3390927314758301, + 0.03252435475587845, + 0.12088879942893982, + 2.8850778361135205, + -0.10702899843454362, + 1.506445050239563, + 0.0 + ], + [ + 0.35250425338745117, + 0.009205148555338383, + 0.10765178501605988, + 2.894632579880305, + -0.11894208937883377, + 1.4516977071762085, + 0.0 + ], + [ + 0.3648030161857605, + -0.008096429519355297, + 0.09344543516635895, + 2.89535445173318, + -0.1120196282863617, + 1.4230073690414429, + 0.0 + ], + [ + 0.37344178557395935, + -0.021240629255771637, + 0.07779324799776077, + 2.9027573337131223, + -0.11142983287572861, + 1.375423789024353, + 0.0 + ], + [ + 0.37459662556648254, + -0.026198405772447586, + 0.05803198367357254, + 2.9381618817620954, + -0.10995618253946304, + 1.3424417972564697, + 0.0 + ], + [ + 0.36835023760795593, + -0.025541353970766068, + 0.05032321810722351, + 2.956313731270381, + -0.1102914586663246, + 1.3277531862258911, + 0.0 + ], + [ + 0.36027202010154724, + -0.02404981665313244, + 0.03634468838572502, + 2.989446314173289, + -0.11109656840562819, + 1.316556453704834, + 0.0 + ], + [ + 0.359230637550354, + -0.024704130366444588, + 0.030506979674100876, + 3.001839133101054, + -0.10525497049093248, + 1.309196949005127, + 0.0 + ], + [ + 0.3584958612918854, + -0.02260643057525158, + 0.030007580295205116, + 3.009941266971179, + -0.11608969420194626, + 1.3143494129180908, + 0.0 + ], + [ + 0.35398557782173157, + -0.019532984122633934, + 0.02727624960243702, + 3.044223094480582, + -0.12396696209907532, + 1.340928316116333, + 0.0 + ], + [ + 0.35167959332466125, + -0.012261503376066685, + 0.028480077162384987, + 3.0671335031562528, + -0.13657575845718384, + 1.369297742843628, + 0.0 + ], + [ + 0.35189753770828247, + -0.008350824005901814, + 0.030047647655010223, + 3.0744525661044797, + -0.1535971611738205, + 1.3761802911758423, + 1.0 + ], + [ + 0.3512072265148163, + -0.006863827351480722, + 0.03019585832953453, + 3.0775652845674237, + -0.16086842119693756, + 1.39474618434906, + 1.0 + ], + [ + 0.3510279953479767, + -0.004059859551489353, + 0.04675820469856262, + 3.0486453046374997, + -0.14888477325439453, + 1.4242743253707886, + 1.0 + ], + [ + 0.3463469445705414, + 0.008149323984980583, + 0.0774770975112915, + 3.025423230724879, + -0.07882881164550781, + 1.4785162210464478, + 1.0 + ], + [ + 0.3381657898426056, + 0.015057893469929695, + 0.11289862543344498, + 2.979333924251147, + -0.08567028492689133, + 1.5062605142593384, + 1.0 + ], + [ + 0.3274476230144501, + 0.03204524517059326, + 0.13786007463932037, + 2.934156986074992, + -0.12075348198413849, + 1.564159631729126, + 1.0 + ], + [ + 0.3195037841796875, + 0.04581454396247864, + 0.15473783016204834, + 2.8917998542361936, + -0.14509262144565582, + 1.6000951528549194, + 1.0 + ], + [ + 0.31936171650886536, + 0.046794552356004715, + 0.15473783016204834, + 2.8917998542361936, + -0.14509262144565582, + 1.6031631231307983, + 1.0 + ], + [ + 0.31904351711273193, + 0.04676813259720802, + 0.15430250763893127, + 2.893348129587718, + -0.14501506090164182, + 1.6029393672943115, + 1.0 + ], + [ + 0.31904351711273193, + 0.04676813259720802, + 0.15430250763893127, + 2.893348129587718, + -0.14501506090164182, + 1.6029393672943115, + 1.0 + ], + [ + 0.3192792236804962, + 0.04677147790789604, + 0.15436245501041412, + 2.8918133844905576, + -0.14504241943359375, + 1.602946400642395, + 1.0 + ], + [ + 0.31904351711273193, + 0.04676813259720802, + 0.15430250763893127, + 2.893348129587718, + -0.14501506090164182, + 1.6029393672943115, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.057769834995269775, + 0.3599643111228943, + 0.5196115374565125, + 0.5262376368045807, + 0.5262376368045807, + 0.5262376368045807, + 0.5262376368045807, + 0.5262376368045807, + 0.5262376368045807, + 0.5262376368045807, + 0.5262376368045807, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.527181476354599, + 0.45868784189224243, + 0.1545853614807129, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "2277", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/2277/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/23977.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/23977.json new file mode 100644 index 00000000..2fd60390 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/23977.json @@ -0,0 +1,831 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Move the ice cream scoop close to the red can and the pot with the metal part of the spoon facing the pot." + ], + "videos": [ + { + "video_path": "videos/train/23977/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + -7.2669591801478275e-09, + -2.6980278800509545e-10, + -0.0015339703798321915, + 1.0 + ], + [ + -0.0034117044856053994, + -0.0015632551154766744, + 0.008018085636895076, + -0.0042536004179281545, + -0.02585308737119574, + -0.002202255758288875, + 1.0 + ], + [ + -0.011707792405469894, + -0.033200572462415444, + -0.0024858198249331272, + 0.05893069014294443, + -0.00014783798856397872, + -0.049154093050560385, + 1.0 + ], + [ + -0.009823416930754942, + -0.023199360780031777, + 0.00011460155587830963, + 0.0538673574332882, + -0.005402210091346852, + -0.0366879075872313, + 1.0 + ], + [ + -0.01637659075041175, + -0.03958175046239678, + 0.00029313421571343335, + 0.0808294504745159, + -0.0008966107889117138, + -0.08585425818601833, + 1.0 + ], + [ + -0.00621827411917322, + -0.022649095535941178, + 0.011446274585110596, + 0.050274245208978524, + -0.014935369573893308, + -0.04407384458830388, + 1.0 + ], + [ + -0.0024800142304733513, + -0.029420465143004743, + 0.01743197385755355, + 0.033785541711364483, + -0.03676365423593231, + -0.07429025556622085, + 1.0 + ], + [ + 0.003906616477120163, + -0.015137641714140483, + 0.020669265814711086, + 0.030586842606357623, + -0.037378425018016255, + -0.0327639119088545, + 1.0 + ], + [ + 0.0015996221346103003, + -0.018804803300599797, + 0.010738511082099304, + 0.039836872976595626, + -0.031757970075370644, + -0.04214681182636097, + 1.0 + ], + [ + -0.0038992369257964815, + -0.024576072200535128, + 0.0018462490785198505, + 0.02713120096435493, + -0.00986933671898212, + -0.07110304970018078, + 1.0 + ], + [ + 0.0002393034863472474, + -0.009306274612223884, + -0.0005119683459033307, + 0.014940961091769378, + -0.00616890312791855, + -0.03480531224247397, + 1.0 + ], + [ + -0.0033220586286767753, + -0.008646625355042126, + 0.017537782874482145, + -0.03635999563164152, + -0.04895827132727228, + -0.04147918687529293, + 1.0 + ], + [ + 0.004484329777526256, + 0.0026026835741736276, + 0.0138657011748836, + -0.049644877806427505, + -0.022182687623943168, + 0.028817662631744484, + 1.0 + ], + [ + 0.005967042227474264, + 0.003797948245750848, + 0.0018120937732919772, + -0.038019059340829414, + -0.0010971705434383193, + 0.03444061528044036, + 1.0 + ], + [ + 0.0009152909085459657, + 0.0022764935486239857, + 0.0006080781211718798, + -0.012946373673293954, + 0.008476237101598384, + 0.017122803675440654, + 0.0 + ], + [ + -0.0031310966398621957, + 0.0032199419700491916, + 0.0014288985775232986, + -0.006756206995112619, + 0.0012511873383507504, + 0.011407706310157691, + 0.0 + ], + [ + -0.006264437064404497, + 0.007610608394530232, + -0.0012784101055193116, + -0.012334715110040426, + -0.021057859192901228, + 0.0032626929734692845, + 0.0 + ], + [ + -0.008398249849800872, + 0.009881457122887996, + -0.005731702793590681, + -0.016400574206933687, + -0.021862674538337113, + -0.08714356164251223, + 0.0 + ], + [ + 0.0030488233157816447, + 0.010279104658700373, + -0.0343376879696058, + -0.03970918534311314, + 0.06980687389868465, + -0.18731659729960898, + 0.0 + ], + [ + 0.0031527655294735536, + 0.019628134489167386, + -0.023450822312500803, + -0.04178390870590941, + 0.04047070219512252, + -0.21815258718885014, + 0.0 + ], + [ + -3.8405942704611214e-05, + 0.017311099426797597, + -0.01431344171030517, + 0.0025393412292679485, + 0.048775732483179984, + -0.21715638682374275, + 0.0 + ], + [ + -0.004515913174366798, + 0.0072460482236062235, + -0.0012743266740816534, + 0.024617449653458183, + 0.04623140756038479, + -0.2536065495549138, + 0.0 + ], + [ + -0.0014897678624592999, + 0.00846732212969535, + -0.0017680606684506296, + -0.020659224541868696, + 0.01447360601489508, + -0.2022033570276806, + 0.0 + ], + [ + 0.005339680367531592, + 0.013787440921087446, + -0.0001638594872966015, + -0.048475808683979243, + 0.007352235030528314, + -0.11646463447523296, + 0.0 + ], + [ + 0.004199308871567003, + 0.009135247904101822, + 0.010620377265332225, + -0.025810472667984818, + 0.00990166354353552, + -0.11500573791058885, + 0.0 + ], + [ + 0.0050196352184674635, + 0.012213829175802072, + 0.01662701826968186, + 0.005704281772948324, + 0.0026313878222510496, + -0.06057465863844752, + 0.0 + ], + [ + 3.1548055141345324e-05, + 0.01000789708874805, + 0.011412106692874468, + 0.013078670809980967, + 0.014664307909974448, + -0.034135068198646444, + 0.0 + ], + [ + -0.002184651879831106, + 0.006478164742675873, + 0.016433226465683518, + 0.02312505594438766, + 0.008005664491753913, + -0.009778394812891122, + 0.0 + ], + [ + 0.00025200027974034666, + 0.001288464397436524, + 0.01725354904160248, + 0.054055765506390664, + 0.028985783804423324, + -0.009380142123452947, + 0.0 + ], + [ + -0.0006129106995242815, + 0.003383314830097263, + -0.0004328923037296961, + -0.0043245969461456445, + 0.020622621758439562, + 0.0032190159334759454, + 0.0 + ], + [ + -0.0008527265914510736, + -0.0006868114858810486, + 0.0017510626439883248, + -0.00047507916676723863, + 0.017693901062176935, + 0.0031275004721048306, + 0.0 + ], + [ + -0.003362895226247861, + -0.006144974627438965, + 0.012455725648211569, + 0.03882106564036934, + -0.005242110322768347, + -0.006252112539444562, + 0.0 + ], + [ + -0.0009338192406769619, + -0.001396594178781302, + 0.002217158265107962, + 0.010319617085641572, + -0.004016279620813444, + -0.008792287523355377, + 0.0 + ], + [ + -0.00015415741515495185, + -9.987099976257861e-05, + 0.0017346861390885534, + 0.012009551855151448, + -0.005831274262853596, + 0.0022310022275801927, + 0.0 + ], + [ + -0.0011205185788884288, + 0.0008578829726519016, + -6.279811004418027e-06, + -0.005362667559039452, + -0.007102305524494885, + -0.006680948393796569, + 1.0 + ], + [ + -0.0026005189423210276, + -0.0003529817535139271, + 0.0002810652216122264, + -0.0018728893194545199, + -0.015340227093832762, + 0.00841063963657364, + 1.0 + ], + [ + -0.0024291013460025915, + -0.0016631740450881406, + -0.0032213752070315814, + 0.020617355465087604, + -0.019918478843673863, + 0.004881354540511915, + 1.0 + ], + [ + 0.00027838203875569584, + 0.00697379247709238, + -0.023001058396055186, + -0.05575202601596121, + -0.010082266326112048, + 0.010646787766991968, + 1.0 + ], + [ + 4.628526899905832e-05, + 0.0002336193508950191, + -0.023131857714001284, + -0.0438805229284585, + -0.016735237046335474, + 0.028950492958993365, + 1.0 + ], + [ + 0.005168646356567719, + -0.0062462751217184104, + -0.00643044802907303, + -0.002668642490857719, + -0.02209555828604108, + 0.0569568192596257, + 1.0 + ], + [ + 0.001161302890878126, + -0.0006824236519496417, + 2.7845323345628496e-05, + 0.0017391457001440676, + 9.747615839202804e-05, + 0.0031400435458578575, + 1.0 + ], + [ + 7.91288208626752e-05, + 0.00023003212224209155, + -1.8427209514061464e-07, + -0.001450540274538859, + 0.0004990295217101359, + -4.0908808337751263e-07, + 1.0 + ] + ], + "state": [ + [ + 0.3230612576007843, + -0.040372107177972794, + 0.11114369332790375, + 3.039052503305026, + -0.24680234491825104, + -0.37196290493011475, + 1.0 + ], + [ + 0.3230612576007843, + -0.040372107177972794, + 0.11114369332790375, + 3.038668194907256, + -0.2466450333595276, + -0.37038937211036677, + 1.0 + ], + [ + 0.3220936059951782, + -0.03921198472380638, + 0.10242068022489548, + 3.034496734040328, + -0.2207024991512299, + -0.3708670139312744, + 1.0 + ], + [ + 0.3237262964248657, + -0.004143453203141689, + 0.09880652278661728, + 3.082612136500426, + -0.21503606438636783, + -0.32085788249969477, + 1.0 + ], + [ + 0.3222207725048065, + 0.02075427584350109, + 0.09526258707046509, + 3.1286025905334434, + -0.20733740925788882, + -0.2837580442428589, + 1.0 + ], + [ + 0.3180748522281647, + 0.06318645179271698, + 0.09110140800476074, + -3.0917301533096513, + -0.20455294847488406, + -0.19609051942825317, + 1.0 + ], + [ + 0.3186742067337036, + 0.08671197295188904, + 0.07974991947412491, + -3.0507595633440694, + -0.1916248649358749, + -0.15049616992473602, + 1.0 + ], + [ + 0.3236633837223053, + 0.11718884110450745, + 0.06485506892204285, + -3.0320880134874066, + -0.1611684262752533, + -0.07220256328582764, + 1.0 + ], + [ + 0.3317813575267792, + 0.13395272195339203, + 0.046835657209157944, + -3.0074266364150724, + -0.12748512625694272, + -0.035271450877189636, + 1.0 + ], + [ + 0.3351074159145355, + 0.15392008423805237, + 0.03897837921977043, + -2.9735430051856717, + -0.10151296108961105, + 0.010964945890009405, + 1.0 + ], + [ + 0.33072733879089355, + 0.17841219902038574, + 0.040861792862415314, + -2.9541440923982343, + -0.10340274870395659, + 0.08310212939977646, + 1.0 + ], + [ + 0.3299833834171295, + 0.18742941319942474, + 0.04311179369688034, + -2.9429852237277707, + -0.10376176238059998, + 0.11863835155963896, + 1.0 + ], + [ + 0.32688236236572266, + 0.1990812122821808, + 0.027363991364836693, + -2.984579252200671, + -0.0638151541352272, + 0.16902142763137817, + 1.0 + ], + [ + 0.3322480022907257, + 0.19958853721618652, + 0.013576555997133255, + -3.0326396991782865, + -0.037379682064056396, + 0.1440175324678421, + 1.0 + ], + [ + 0.3387446105480194, + 0.19691486656665802, + 0.011586768552660942, + -3.069446923332759, + -0.0325229950249195, + 0.10988254845142363, + 1.0 + ], + [ + 0.3399229943752289, + 0.1948046237230301, + 0.01084634568542242, + -3.081826379644223, + -0.03973789140582085, + 0.09217984974384308, + 0.0 + ], + [ + 0.3371596932411194, + 0.1914069950580597, + 0.009104556404054165, + -3.0881307517462453, + -0.040302857756614685, + 0.08070846647024155, + 0.0 + ], + [ + 0.3315039873123169, + 0.18325644731521606, + 0.009721343405544758, + -3.1003677790337285, + -0.01910065114498138, + 0.07857594639062881, + 0.0 + ], + [ + 0.3238254189491272, + 0.1725112795829773, + 0.014879557304084301, + -3.1185959951304874, + -0.0007695025415159762, + 0.16652886569499972, + 0.0 + ], + [ + 0.32864049077033997, + 0.1620994210243225, + 0.048974137753248215, + 3.1243974567675075, + -0.07482744008302689, + 0.3522275090217591, + 0.0 + ], + [ + 0.3365538716316223, + 0.14452871680259705, + 0.07292812317609787, + 3.0666839500241956, + -0.10977901518344879, + 0.5722243189811707, + 0.0 + ], + [ + 0.34385547041893005, + 0.12997235357761383, + 0.08839903771877289, + 3.0467865039878568, + -0.13963045179843903, + 0.7944362759590149, + 0.0 + ], + [ + 0.3455919623374939, + 0.12161777913570404, + 0.08970604836940765, + 3.0386300479346, + -0.15729427337646484, + 1.0544061660766602, + 0.0 + ], + [ + 0.35182836651802063, + 0.11591131240129471, + 0.09206920117139816, + 2.988198073702403, + -0.1478109806776047, + 1.2592862844467163, + 0.0 + ], + [ + 0.3662906587123871, + 0.11645205318927765, + 0.09509942680597305, + 2.9235219677262982, + -0.13631933927536008, + 1.3765983581542969, + 0.0 + ], + [ + 0.3783193528652191, + 0.11948870867490768, + 0.08735548704862595, + 2.88351887662942, + -0.12026546895503998, + 1.4918553829193115, + 0.0 + ], + [ + 0.3948378264904022, + 0.1247406005859375, + 0.0750921368598938, + 2.882540615397044, + -0.10714739561080933, + 1.5514363050460815, + 0.0 + ], + [ + 0.40745094418525696, + 0.12543347477912903, + 0.06667641550302505, + 2.891815351443835, + -0.11250770092010497, + 1.5884209871292114, + 0.0 + ], + [ + 0.4177984893321991, + 0.12505286931991577, + 0.05219975486397743, + 2.913654090958186, + -0.11784046888351442, + 1.599954605102539, + 0.0 + ], + [ + 0.42288631200790405, + 0.12739421427249908, + 0.03582780435681343, + 2.965777265029498, + -0.1439422070980072, + 1.6158016920089722, + 0.0 + ], + [ + 0.42617231607437134, + 0.12678883969783783, + 0.03674735501408577, + 2.9613511731200894, + -0.16480989754199984, + 1.6162458658218384, + 0.0 + ], + [ + 0.42583537101745605, + 0.12623462080955505, + 0.0347866453230381, + 2.960831644135066, + -0.1827777922153473, + 1.6163429021835327, + 0.0 + ], + [ + 0.42207375168800354, + 0.12518326938152313, + 0.021040381863713264, + 2.9986911584907254, + -0.1764945983886719, + 1.6216328144073486, + 0.0 + ], + [ + 0.4210338890552521, + 0.1246306449174881, + 0.01852014847099781, + 3.007565604644366, + -0.17126122117042542, + 1.629884123802185, + 0.0 + ], + [ + 0.4211580157279968, + 0.12478159368038177, + 0.016786746680736542, + 3.0200901572876653, + -0.16577959060668945, + 1.6268523931503296, + 0.0 + ], + [ + 0.4220704138278961, + 0.12370768189430237, + 0.016710538417100906, + 3.013762133317538, + -0.1579171121120453, + 1.6326957941055298, + 1.0 + ], + [ + 0.42191246151924133, + 0.12117581814527512, + 0.01598183438181877, + 3.013517217832156, + -0.14376582205295563, + 1.6222918033599854, + 1.0 + ], + [ + 0.4199999272823334, + 0.11824212968349457, + 0.018585605546832085, + 3.0351789613538465, + -0.12463003396987912, + 1.6148496866226196, + 1.0 + ], + [ + 0.42460399866104126, + 0.11578361690044403, + 0.042048729956150055, + 2.9808875938230237, + -0.11572709679603575, + 1.6031142473220825, + 1.0 + ], + [ + 0.4212188422679901, + 0.11307788640260696, + 0.06477227061986923, + 2.9406914134793958, + -0.10378248989582062, + 1.5716928243637085, + 1.0 + ], + [ + 0.4138108491897583, + 0.11768843978643417, + 0.07033517956733704, + 2.9446038772636136, + -0.09330130368471144, + 1.5112167596817017, + 1.0 + ], + [ + 0.4132179915904999, + 0.11889712512493134, + 0.0702832043170929, + 2.9466303308778485, + -0.09401101619005203, + 1.5081429481506348, + 1.0 + ], + [ + 0.41344785690307617, + 0.11895743012428284, + 0.07033517956733704, + 2.945170613127299, + -0.09450051188468933, + 1.5082404613494873, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.052336275577545166, + 0.3368593454360962, + 0.5806747674942017, + 0.5922945439815521, + 0.5922945439815521, + 0.5922945439815521, + 0.5922945439815521, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5931822955608368, + 0.5940692126750946, + 0.5384550094604492, + 0.22988241910934448, + 0.00520777702331543, + 0.0, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516 + ], + "episode_id": "23977", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/23977/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/346.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/346.json new file mode 100644 index 00000000..5a927914 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/346.json @@ -0,0 +1,603 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "put banana in pot cardboard fence" + ], + "videos": [ + { + "video_path": "videos/train/346/rgb.mp4" + } + ], + "action": [ + [ + -0.00011799907518038634, + 0.00017852040799841174, + -1.4162521192420834e-07, + -0.001125234354924498, + -0.0007449618380323638, + -0.0007288755714804661, + 1.0 + ], + [ + -0.003862288184764138, + 0.0007500682019763629, + 0.005009947141436263, + 0.0007748246765351693, + -0.0243584071258082, + 0.021543328775349323, + 1.0 + ], + [ + 0.007831810358466279, + -0.013490250025100662, + -0.006066781859178772, + -0.057959711661628, + -0.020357476762181845, + 0.12274095999502654, + 1.0 + ], + [ + 0.010578199758379743, + -0.019861088826281632, + -0.01580535439025533, + -0.05176352791550576, + 0.03313618085399437, + 0.06252777434289702, + 1.0 + ], + [ + 0.01785508921628847, + -0.010392813260455313, + 0.0013375420160227475, + -0.03155403471413408, + -0.02282885987373748, + 0.10213190713220881, + 1.0 + ], + [ + 0.021676895637801924, + -0.012662937426923255, + 0.014807851850910811, + -0.015075243599023225, + 0.0021227083114920948, + 0.024159733373068898, + 1.0 + ], + [ + 0.01732876983268316, + -0.008739152412545978, + 0.017900601662753295, + 0.0018086006623594195, + 0.021217024236632746, + 0.011415518506383431, + 1.0 + ], + [ + 0.012697673827784745, + -0.0013073018747422056, + 0.01585347564751782, + 0.012890481305066872, + 0.0464607646811306, + 5.722459521662456e-05, + 1.0 + ], + [ + 0.006597696468764818, + 0.001797837444144988, + 0.021843261274228733, + 0.006878871401702641, + 0.03690806801987771, + 0.011203167452661738, + 1.0 + ], + [ + 0.0008844706815405846, + 0.0026885071444586912, + 0.018923252395470037, + 0.009464702282468141, + -0.026237070618220372, + 0.007033815914049922, + 1.0 + ], + [ + 0.0028504580006673483, + 0.0001273228646752178, + 0.012285158755846539, + 0.024546480985417243, + -0.027124465919119937, + 0.002537251428633804, + 1.0 + ], + [ + 0.005744151392172613, + -0.002364281111801664, + 0.0023282661413751276, + 0.014775913050168722, + 0.009730678014174588, + -0.0002684254941437432, + 1.0 + ], + [ + 0.0006954296886782774, + -0.0008177058137561576, + 0.002139730350535911, + -0.026918340986643776, + 0.023315374378609314, + -0.007924586687935355, + 1.0 + ], + [ + -0.005523693631126991, + 0.0009629971544678167, + -0.0016396193524543538, + -0.012855565020082445, + -0.01264935859550317, + 0.020478818630408305, + 0.0 + ], + [ + -0.005103893379415184, + -0.0038010962100459055, + -0.006677207366552649, + 0.0006048683562208886, + -0.013802578797496858, + -0.02619934235334942, + 0.0 + ], + [ + 0.0012263907026388906, + 0.0004459067845875881, + -0.015638500385143386, + -0.02053853811072358, + 0.00650604417318742, + -0.022325181630032324, + 0.0 + ], + [ + 0.0018169078923207108, + 0.007022250505084949, + -0.02774937377519867, + -0.03507265617030866, + 0.03558019247400147, + -0.03261930597089279, + 0.0 + ], + [ + 0.0043185944367882525, + 0.003625175420740897, + -0.021175136082859165, + -0.008395678914628137, + 0.05398499836669642, + -0.006542591330046829, + 0.0 + ], + [ + -0.0028245411592124512, + 0.004618739809555407, + -0.015210162750132806, + -0.02598340274303503, + 0.01808193098289891, + 0.009628061655244644, + 0.0 + ], + [ + -0.01588742494354134, + 0.0026701575499412282, + -0.0022029948512770903, + -0.0037141190879101064, + -0.018029180696642867, + 0.036825730407102186, + 0.0 + ], + [ + -0.01797364728350155, + 0.001219562270051552, + 0.003963554666708154, + 0.0025582487748342514, + -0.01804526959942076, + 0.04132152154082789, + 0.0 + ], + [ + -0.015451859910410417, + -0.0027229230082726355, + 0.009848545895156938, + 0.01602210412050608, + -0.014225754700112385, + 0.028510160417846497, + 0.0 + ], + [ + -0.01404632370944586, + -0.0030192503059641306, + 0.010847235819967388, + 0.015589223963155024, + -0.0213201136440856, + 0.030265812765085717, + 0.0 + ], + [ + -0.011658781463087804, + -0.0017766002433637453, + 0.006168528301209859, + 0.027989854521572266, + 0.006541200090002691, + 0.032666791920631495, + 0.0 + ], + [ + -0.0023818447028394426, + -0.004450271215325108, + 0.0012073181527072396, + 0.006308982812840074, + -0.0032187175664498777, + -0.004893512443377819, + 0.0 + ], + [ + -0.005629401931995096, + -0.0017974081206813624, + -9.411790263650962e-05, + -0.009789606046823243, + -0.028601840134821616, + -0.014249147230190017, + 0.0 + ], + [ + 0.0014038869678525269, + -0.00392393862721984, + -0.013854098397708462, + -0.016150563376433145, + 0.01170512274263221, + -0.026507228526359835, + 1.0 + ], + [ + 0.004309494295161061, + 0.00027295878035536156, + -0.023300284589294985, + -0.019524985005100835, + 0.0012715849508721003, + -0.03599541165645559, + 1.0 + ], + [ + 0.006210740781385351, + 0.0058642452745778, + -0.015658888370047395, + -0.011755588818268683, + -0.02121261519641501, + -0.009047101559679499, + 1.0 + ], + [ + -0.0003158409023800665, + 0.0016393900900788809, + -0.0003650781566801689, + -0.005780240140698366, + -0.004610490040938734, + -0.0006740038885510416, + 1.0 + ] + ], + "state": [ + [ + 0.25642186403274536, + -0.003085855394601822, + 0.08004199713468552, + -2.9066125770383557, + -0.08274048566818239, + 0.27030512690544134, + 1.0 + ], + [ + 0.25635820627212524, + -0.0032836843747645617, + 0.07999096065759659, + -2.907811002927371, + -0.08218566328287126, + 0.2711904048919678, + 1.0 + ], + [ + 0.25293341279029846, + -0.0037886379286646843, + 0.07464350759983063, + -2.905734511213847, + -0.0534898266196251, + 0.25586068630218506, + 1.0 + ], + [ + 0.25707048177719116, + 0.009386355988681316, + 0.08410046994686127, + -2.959105970459529, + -0.004723940044641495, + 0.14134737849235535, + 1.0 + ], + [ + 0.26510608196258545, + 0.027360664680600166, + 0.10329744219779968, + -3.0108698328309735, + -0.025961145758628845, + 0.07384535670280458, + 1.0 + ], + [ + 0.28213268518447876, + 0.039127416908741, + 0.10378950089216232, + -3.040476659434386, + 0.01009033340960741, + -0.02440032921731472, + 1.0 + ], + [ + 0.30401015281677246, + 0.05269070714712143, + 0.09011749178171158, + -3.055826189117976, + 0.010414034128189087, + -0.04865223169326783, + 1.0 + ], + [ + 0.3216377794742584, + 0.06208495795726776, + 0.07285176217556, + -3.05414308805997, + -0.009748061187565327, + -0.06184133514761925, + 1.0 + ], + [ + 0.3346291780471802, + 0.0639725923538208, + 0.057297565042972565, + -3.041118437545844, + -0.056026078760623925, + -0.065961092710495, + 1.0 + ], + [ + 0.34245312213897705, + 0.06385886669158936, + 0.03578796610236168, + -3.0333469604426107, + -0.09161804616451262, + -0.08086422085762023, + 1.0 + ], + [ + 0.3450222611427307, + 0.06302019208669662, + 0.016846055164933205, + -3.0234646146469792, + -0.06477388739585875, + -0.0850289687514305, + 1.0 + ], + [ + 0.34875646233558655, + 0.06402811408042908, + 0.004841627553105354, + -2.9989184011989316, + -0.03753945976495743, + -0.0843508094549179, + 1.0 + ], + [ + 0.3547748327255249, + 0.06620009988546371, + 0.0030901655554771423, + -2.9840937723689756, + -0.04720940813422204, + -0.08546996116638184, + 1.0 + ], + [ + 0.3556578755378723, + 0.06727182865142822, + 0.0011402055388316512, + -3.0111675580316266, + -0.07147869467735289, + -0.08129163831472397, + 1.0 + ], + [ + 0.34996482729911804, + 0.06656371802091599, + 0.0022423341870307922, + -3.022700773673602, + -0.05626108124852181, + -0.09997905790805817, + 0.0 + ], + [ + 0.34479594230651855, + 0.07007946074008942, + 0.009025057777762413, + -3.0236864780359944, + -0.04564156383275986, + -0.07230066508054732, + 0.0 + ], + [ + 0.34514865279197693, + 0.06776551902294159, + 0.024542365223169327, + -3.0452276562624654, + -0.054717957973480225, + -0.05086439475417137, + 0.0 + ], + [ + 0.3449976444244385, + 0.058101095259189606, + 0.051546383649110794, + -3.081875069188424, + -0.09324795752763747, + -0.021714638918638226, + 0.0 + ], + [ + 0.34724289178848267, + 0.05316869169473648, + 0.07277864217758179, + -3.090495044487067, + -0.1475265920162201, + -0.01837738789618015, + 0.0 + ], + [ + 0.3421526551246643, + 0.04787176102399826, + 0.08715542405843735, + -3.114902440580078, + -0.16508489847183228, + -0.02906009182333946, + 0.0 + ], + [ + 0.3260585367679596, + 0.04561043903231621, + 0.08664652705192566, + -3.112596336873718, + -0.14596998691558838, + -0.06577590852975845, + 0.0 + ], + [ + 0.30882206559181213, + 0.045639291405677795, + 0.08007748425006866, + -3.1040795018249234, + -0.1266127377748489, + -0.10687893629074097, + 0.0 + ], + [ + 0.2951338589191437, + 0.05021597445011139, + 0.06846477091312408, + -3.084516519802161, + -0.1112782284617424, + -0.13500644266605377, + 0.0 + ], + [ + 0.2829631567001343, + 0.055535633116960526, + 0.05631352216005325, + -3.0657057260447225, + -0.08821968734264374, + -0.1641126573085785, + 0.0 + ], + [ + 0.2723945081233978, + 0.05955546721816063, + 0.0492936335504055, + -3.0348304008417806, + -0.09221754223108292, + -0.1973220407962799, + 0.0 + ], + [ + 0.2710270881652832, + 0.06447254121303558, + 0.04835115745663643, + -3.0290038903527936, + -0.08953734487295151, + -0.19209247827529904, + 0.0 + ], + [ + 0.2658363878726959, + 0.06729080528020859, + 0.04814206808805466, + -3.0403207187824925, + -0.062703937292099, + -0.17469173669815063, + 0.0 + ], + [ + 0.2667762339115143, + 0.06966681778430939, + 0.06238199397921562, + -3.058079460757323, + -0.07700873166322707, + -0.1494310200214386, + 1.0 + ], + [ + 0.2689305543899536, + 0.06710188090801239, + 0.0858410969376564, + -3.0804178473823747, + -0.08122848719358444, + -0.11354891955852509, + 1.0 + ], + [ + 0.27307817339897156, + 0.059774138033390045, + 0.10156575590372086, + -3.0930023846500596, + -0.06060432270169258, + -0.10320495069026946, + 1.0 + ], + [ + 0.2725768983364105, + 0.05816201493144035, + 0.10183112323284149, + -3.09883655061061, + -0.056031987071037286, + -0.10230640321969986, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.012049198150634766, + 0.012049198150634766, + 0.011358082294464111, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.012049198150634766, + 0.06485205888748169, + 0.3208107352256775, + 0.5922945439815521, + 0.6338784694671631, + 0.6063922345638275, + 0.6055178344249725, + 0.6055178344249725, + 0.6055178344249725, + 0.6055178344249725, + 0.6055178344249725, + 0.6055178344249725, + 0.6055178344249725, + 0.6055178344249725, + 0.5365830659866333, + 0.2493598461151123, + 0.025468170642852783, + 0.012049198150634766, + 0.012741804122924805, + 0.012741804122924805 + ], + "episode_id": "346", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/346/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/4682.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/4682.json new file mode 100644 index 00000000..b547b1df --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/4682.json @@ -0,0 +1,584 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Place the sushi inside the pot" + ], + "videos": [ + { + "video_path": "videos/train/4682/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + -1.0774416864369814e-09, + -1.5932344358333804e-09, + -0.0015339864448118735, + 1.0 + ], + [ + -0.0007094465131767068, + 0.000301790001504133, + 0.009112311782273994, + -0.016758479121634456, + -0.02386558079792943, + -0.0012181099518608443, + 1.0 + ], + [ + 0.013554717013019474, + -0.002080559023620714, + 0.014598877429243654, + -0.025960959413227493, + -0.026248790719438246, + 0.03485935810519088, + 1.0 + ], + [ + 0.02526996914449951, + -0.0022387873530728773, + 0.011981054584064796, + 0.013243635397901705, + 0.017755654397865767, + 0.05286648835302978, + 1.0 + ], + [ + 0.02545033411516179, + -0.006192459074121539, + 0.015686646224700836, + 0.022094423751745713, + 0.025506365902097886, + 0.017992512105274173, + 1.0 + ], + [ + 0.0136704700642176, + -0.004706291207751386, + 0.014843271433360929, + 0.046860140221938476, + -0.021222200793150552, + -0.024052877805098814, + 1.0 + ], + [ + 0.011194400811402164, + -0.009859261988471675, + 0.012714729537933149, + 0.02258837908630905, + -0.019140616334167696, + -0.042363459516456106, + 1.0 + ], + [ + 0.0073369141662703065, + -0.007158193763938788, + 0.01051389597864101, + 0.008305617533721329, + -0.02682954726607507, + -0.0314392443956201, + 1.0 + ], + [ + 0.006122883120999592, + -0.009353173592068208, + 0.004748364928134408, + 0.009426059580078206, + -0.007596479147105749, + -0.015572139994521525, + 1.0 + ], + [ + -5.140715723884528e-05, + -0.005002165576724554, + 0.005713724026298917, + -0.0077245469347651825, + -0.0042462204441477, + -0.023665514719491215, + 1.0 + ], + [ + -0.0001584221510302298, + 0.0027553635284802295, + 0.0005094400180720931, + -0.013796874855894948, + 0.007194469910557962, + 0.0009097708412044518, + 0.0 + ], + [ + -0.00315524213675128, + 0.0070090826185382395, + -0.002453790199588924, + -0.029689602267662055, + 0.015724027050325166, + 0.002807763732129511, + 0.0 + ], + [ + -0.0015037526819682743, + 0.009045247454100597, + -0.024799998801851582, + -0.021169436409362863, + 0.07416772225975815, + -0.0096158192360683, + 0.0 + ], + [ + -0.00953519713677339, + 0.014782128045393465, + -0.027599078334417125, + -0.028073913300815337, + 0.07494648291619813, + -0.005735319419812942, + 0.0 + ], + [ + -0.023737488515726914, + 0.004081520519236222, + -0.0170647728108241, + -0.024597435011416074, + 0.02916417356798834, + -0.02251745261273922, + 0.0 + ], + [ + -0.0227051225438373, + -0.013020844536068568, + -0.007554958990729755, + -0.002762885186291841, + -0.05159677273779236, + -0.008816857941570989, + 0.0 + ], + [ + -0.024541066116466292, + -0.011539712509225128, + -0.0006386529596235153, + 0.030806045452905384, + -0.08964430911310432, + -0.01786640574625395, + 0.0 + ], + [ + -0.023800605510053342, + -0.007686940616932584, + 0.005074378981235873, + 0.02920517417714557, + -0.06713358428814081, + -0.0014748597819653436, + 0.0 + ], + [ + -0.01544460905360122, + 0.00295953206943756, + 0.0021562588869260854, + 0.01093771980599255, + -0.011408661518770201, + 0.01139659036774366, + 0.0 + ], + [ + -0.007176721251410678, + 0.005109241329487431, + 0.0028971371406839503, + 0.006832917430080033, + -0.010951862993352672, + 0.017663790809094235, + 0.0 + ], + [ + 0.002338799928374219, + 0.0019646267354465306, + -0.0028331101833664225, + 0.015225257924673002, + 0.01080904222581567, + 0.03242775182543951, + 1.0 + ], + [ + 0.002550235159835577, + 0.0030969717403583285, + -0.0001590643246551537, + -0.006026755893759468, + 0.010912829845923544, + 0.00983648071181706, + 1.0 + ], + [ + 0.006715206717685889, + 0.011126809609206579, + -0.004628010037593772, + -0.018556930393757963, + 0.018552954027733076, + 0.022239205456609765, + 1.0 + ], + [ + 0.01353565187184336, + 0.013934520436694837, + -0.016617443200472037, + -0.014847741957411034, + 0.06093846096162105, + 0.010582021945021605, + 1.0 + ], + [ + 0.005677401093809816, + 0.0047777679522973526, + -0.0037615760171514916, + -0.0004193676689893956, + 0.03044373910202793, + -0.030270913978237016, + 1.0 + ], + [ + 0.002021133912158814, + 0.0011213805458593765, + 0.005548295538893334, + 0.017500469212037253, + 0.023320566353076944, + -0.07931645636888503, + 1.0 + ], + [ + 0.0009794775228823644, + -0.00045512034777013267, + -0.006794229526241307, + 0.010438561448400203, + 0.02890460768189917, + -0.0337401928642882, + 1.0 + ], + [ + -0.0014986972005551899, + -0.0005980331037275852, + -3.366494592949635e-05, + 0.0011135636369596406, + -0.008578041810107111, + 0.009924792663582177, + 1.0 + ], + [ + -7.799647369285533e-05, + -0.00019792407481042496, + -1.403175121809283e-07, + 0.0012477770730885546, + -0.0004928775337392778, + 0.0007434704578862905, + 1.0 + ] + ], + "state": [ + [ + 0.290576696395874, + 0.09409966319799423, + 0.11820695549249649, + 3.1099106801026544, + -0.06290210783481598, + -0.17374612390995026, + 1.0 + ], + [ + 0.290576696395874, + 0.09409966319799423, + 0.11820695549249649, + 3.1098141466551503, + -0.06285344064235687, + -0.17220987379550934, + 1.0 + ], + [ + 0.2903408408164978, + 0.09354064613580704, + 0.10908223688602448, + 3.0930177589231214, + -0.038961201906204224, + -0.17175059020519257, + 1.0 + ], + [ + 0.3044835031032562, + 0.09247715771198273, + 0.09493867307901382, + 3.068477401631423, + -0.014410260133445261, + -0.20783574879169464, + 1.0 + ], + [ + 0.32965800166130066, + 0.08855578303337097, + 0.08319147676229477, + 3.0825475995712956, + -0.03595965355634689, + -0.2592870891094208, + 1.0 + ], + [ + 0.3561461567878723, + 0.086967334151268, + 0.06809209287166595, + 3.1052235459261617, + -0.062478393316268914, + -0.2757693529129029, + 1.0 + ], + [ + 0.3713095486164093, + 0.08700335770845413, + 0.0539703406393528, + -3.1325536613636693, + -0.04037880897521973, + -0.2524906396865845, + 1.0 + ], + [ + 0.3851252794265747, + 0.09373932331800461, + 0.04180743172764778, + -3.1116887164586267, + -0.021585263311862946, + -0.20995360612869265, + 1.0 + ], + [ + 0.3940734565258026, + 0.09946957975625992, + 0.031673017889261246, + -3.1040833314233502, + 0.00430363928899169, + -0.17773693799972534, + 1.0 + ], + [ + 0.40176519751548767, + 0.10776500403881073, + 0.027252431958913803, + -3.0945927036278924, + 0.011310296133160591, + -0.16189028322696686, + 1.0 + ], + [ + 0.40250203013420105, + 0.11297950893640518, + 0.021780963987112045, + -3.102060569571801, + 0.014436851255595686, + -0.13804908096790314, + 1.0 + ], + [ + 0.40196019411087036, + 0.11029545962810516, + 0.021165376529097557, + -3.1158736777775964, + 0.007283947896212339, + -0.139242485165596, + 0.0 + ], + [ + 0.3978707194328308, + 0.10372963547706604, + 0.023461030796170235, + 3.137601683969148, + -0.008362711407244205, + -0.14245343208312988, + 0.0 + ], + [ + 0.39490655064582825, + 0.09511702507734299, + 0.04828348755836487, + 3.1163382680439433, + -0.08249105513095857, + -0.13253465294837954, + 0.0 + ], + [ + 0.38134250044822693, + 0.08272026479244232, + 0.07536627352237701, + 3.087557481723376, + -0.15726634860038755, + -0.12483139336109161, + 0.0 + ], + [ + 0.3550054430961609, + 0.0828464925289154, + 0.08869583159685135, + 3.0591193457418164, + -0.18512582778930664, + -0.1003628745675087, + 0.0 + ], + [ + 0.3329806327819824, + 0.09873215854167938, + 0.09086279571056366, + 3.0554054101281842, + -0.13297635316848755, + -0.09579610824584961, + 0.0 + ], + [ + 0.3099198043346405, + 0.11255314201116562, + 0.08725511282682419, + 3.084532143669673, + -0.04212105646729469, + -0.08576567471027374, + 0.0 + ], + [ + 0.2870912551879883, + 0.1219281479716301, + 0.08075328171253204, + 3.113707998889037, + 0.024987330660223957, + -0.08812299370765685, + 0.0 + ], + [ + 0.27139413356781006, + 0.12028475850820541, + 0.07906690239906311, + 3.12435305503244, + 0.036072101444005966, + -0.09984013438224794, + 0.0 + ], + [ + 0.2636428475379944, + 0.11587698757648468, + 0.07651892304420471, + 3.1305432712012013, + 0.04671210050582886, + -0.11770855635404588, + 0.0 + ], + [ + 0.26586827635765076, + 0.11366714537143707, + 0.07926124334335327, + -3.1389208986708006, + 0.03552110120654107, + -0.15003333985805511, + 1.0 + ], + [ + 0.2679305970668793, + 0.11022280901670456, + 0.07932136952877045, + 3.137887217691638, + 0.02463286183774471, + -0.15990135073661804, + 1.0 + ], + [ + 0.27290278673171997, + 0.09816776216030121, + 0.08382376283407211, + 3.1187845320724925, + 0.005991584155708551, + -0.18206821382045743, + 1.0 + ], + [ + 0.2838604748249054, + 0.08237069845199585, + 0.10067327320575714, + 3.103840200352021, + -0.05517259240150452, + -0.19125284254550934, + 1.0 + ], + [ + 0.2883320450782776, + 0.07678649574518204, + 0.10491958260536194, + 3.1016846020990094, + -0.08442503213882446, + -0.15975588560104373, + 1.0 + ], + [ + 0.2905646860599518, + 0.07506759464740753, + 0.09961046278476715, + 3.112511560442396, + -0.1042935848236084, + -0.07915208488702774, + 1.0 + ], + [ + 0.29088395833969116, + 0.07569680362939835, + 0.10645372420549393, + 3.1193259229236325, + -0.13214239478111267, + -0.04429541900753975, + 1.0 + ], + [ + 0.2894236147403717, + 0.07636075466871262, + 0.10627642273902893, + 3.1217825693362435, + -0.12378066033124924, + -0.054486356675624854, + 1.0 + ], + [ + 0.28935757279396057, + 0.0765625387430191, + 0.10626304149627686, + 3.1231240407847842, + -0.12330257147550583, + -0.055245205760002136, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.04620790481567383, + 0.3218126893043518, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.3810555934906006, + 0.31079989671707153, + 0.03643929958343506, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "4682", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/4682/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/5873.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/5873.json new file mode 100644 index 00000000..c73a6bd3 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/5873.json @@ -0,0 +1,318 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "flip pot upright which is in sink" + ], + "videos": [ + { + "video_path": "videos/train/5873/rgb.mp4" + } + ], + "action": [ + [ + -0.00082000263294433, + -7.466453782392899e-05, + 0.0015614318013789901, + 0.0029874162263894425, + -0.0051927332507799686, + 0.0028530769883012594, + 1.0 + ], + [ + -0.00018491269081222982, + 0.0026035902344583757, + 0.012209279762011376, + 0.1087268001838756, + -0.010266900279530224, + 0.056581973305013945, + 1.0 + ], + [ + -0.0021880775332560436, + 0.0032266125136424673, + 0.0038847041042033496, + 0.15966260548563593, + 0.04319476982551402, + 0.09522199619289701, + 1.0 + ], + [ + -0.0036292868055547585, + 0.00367922307947352, + 0.007406652027010706, + 0.08049906316361147, + 0.035081973566848355, + 0.042119839775207855, + 1.0 + ], + [ + -0.004503582749953799, + -0.00024846193130534844, + 0.01195964918548392, + 0.048366643005682115, + 0.01595624183654295, + 0.004198941931345814, + 1.0 + ], + [ + -0.004845523943243744, + 0.0023213068613341524, + 0.011303078719697896, + 0.019341920238366946, + 0.018030367339306217, + 0.026223023900620358, + 1.0 + ], + [ + -0.004114971142783192, + -0.006443947135325001, + 0.006915183213654191, + -0.012894487352749721, + 0.030733522108376748, + -0.004110886699246275, + 1.0 + ], + [ + -0.0021712507033200064, + -0.004632216160245966, + 0.007101646800809974, + -0.003153151260741384, + 0.0004843764841172617, + 0.00837989380779611, + 1.0 + ], + [ + 0.00021605781766817654, + -0.011888665751510513, + 0.0027128421519854087, + -0.016103731946888383, + 0.014933186412824005, + 0.0060081611078627245, + 1.0 + ], + [ + 0.0016086653553478189, + -0.0098805641027343, + 0.0035118585331301295, + -0.04822851039900436, + 0.01331371171926694, + 0.002411225231370376, + 0.0 + ], + [ + -0.0038818540885260304, + -0.001874976644794594, + 0.007077943320518391, + -0.09440959844196783, + -0.04998897707110319, + -0.029764930759144437, + 0.0 + ], + [ + 0.000552967214453235, + -0.0029430554277015387, + -0.00017115774483648287, + -0.13261304830116177, + 0.003984288287198014, + -0.054291534664374246, + 0.0 + ], + [ + -0.00042355282659887133, + -0.0022870283162429532, + 0.002320310596820844, + -0.18376899086869744, + 0.06853868378976727, + -0.07750305078081861, + 0.0 + ], + [ + -0.0012124504668198535, + -0.0028859505698711047, + 0.0038334698840275516, + -0.0813903158823298, + 0.039279395974008674, + -0.047221797194660894, + 1.0 + ], + [ + 0.000249009348226328, + 0.0008312523240148902, + -0.0004399960096770226, + -0.0032604023027053325, + 0.002675032339824997, + 0.001002497157853473, + 1.0 + ] + ], + "state": [ + [ + 0.2852519452571869, + -0.09647413343191147, + 0.17865513265132904, + -2.9550224413448056, + -0.11350087076425552, + 0.07400803267955779, + 1.0 + ], + [ + 0.2845843434333801, + -0.09615963697433472, + 0.17705155909061432, + -2.9518231471353253, + -0.10786884278059006, + 0.07215694338083267, + 1.0 + ], + [ + 0.28575950860977173, + -0.09632913768291473, + 0.1646229773759842, + -2.8375690301233014, + -0.08696263283491133, + 0.018330661579966545, + 1.0 + ], + [ + 0.28402093052864075, + -0.09827703982591629, + 0.15977811813354492, + -2.6702621599012097, + -0.09925844520330429, + -0.08593045175075531, + 1.0 + ], + [ + 0.2812466025352478, + -0.09795264154672623, + 0.15118947625160217, + -2.5848505218797406, + -0.11125873029232027, + -0.13970831036567688, + 1.0 + ], + [ + 0.2788258194923401, + -0.09101726859807968, + 0.14072918891906738, + -2.5351074059777936, + -0.12257836759090424, + -0.1517949402332306, + 1.0 + ], + [ + 0.2760401666164398, + -0.0860031396150589, + 0.1296059787273407, + -2.512084337072917, + -0.12238742411136627, + -0.1838570088148117, + 1.0 + ], + [ + 0.2739369571208954, + -0.07617289572954178, + 0.1273212432861328, + -2.5228900035196027, + -0.1496349722146988, + -0.1987945884466171, + 1.0 + ], + [ + 0.2738437354564667, + -0.06810320168733597, + 0.12393354624509811, + -2.5249902327829084, + -0.14516563713550568, + -0.20597757399082184, + 1.0 + ], + [ + 0.2756975293159485, + -0.056979112327098846, + 0.128577321767807, + -2.5390982945733747, + -0.15386117994785312, + -0.21967603266239166, + 1.0 + ], + [ + 0.2790519595146179, + -0.04734761640429497, + 0.13149769604206085, + -2.58581659396226, + -0.16345712542533872, + -0.22933608293533325, + 0.0 + ], + [ + 0.2773292064666748, + -0.04147510603070259, + 0.12590955197811127, + -2.6887597759538373, + -0.1364695280790329, + -0.17721503973007202, + 0.0 + ], + [ + 0.2781287431716919, + -0.039005838334560394, + 0.12741291522979736, + -2.828398587303706, + -0.16364434361457822, + -0.12950780987739563, + 0.0 + ], + [ + 0.2783304452896118, + -0.036117035895586014, + 0.12586118280887604, + -3.020703787105628, + -0.25248235464096075, + -0.07538015395402908, + 0.0 + ], + [ + 0.2782715857028961, + -0.03277593106031418, + 0.12221045792102814, + -3.113114571320363, + -0.29693639278411865, + -0.03135182335972785, + 1.0 + ], + [ + 0.278361439704895, + -0.033622607588768005, + 0.12268125265836716, + -3.1160446909540376, + -0.2995816171169281, + -0.03248034417629242, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.18319201469421387, + 0.5024297535419464, + 0.7347722351551056, + 0.8976205363869667, + 0.8567025810480118, + 0.6919880211353302, + 0.3820592164993286, + 0.06803375482559204 + ], + "episode_id": "5873", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/5873/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/83.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/83.json new file mode 100644 index 00000000..9ea8adcc --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/83.json @@ -0,0 +1,584 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Put the metal pot on the blue cloth." + ], + "videos": [ + { + "video_path": "videos/train/83/rgb.mp4" + } + ], + "action": [ + [ + 4.774149852778847e-06, + 0.00021952306442777565, + -1.553402224619463e-07, + -0.0013844133353417248, + 2.7162349958670412e-05, + -0.002194109714208664, + 1.0 + ], + [ + -0.0023385938859878685, + -0.0010978842613608075, + 0.012520116766907466, + -0.000271537430827922, + -0.03506380820310032, + -0.005724298583177262, + 1.0 + ], + [ + 0.014424328176828724, + -0.016258910134085158, + 0.00011861713926078793, + -0.019211557691389717, + 0.02580023157871661, + -0.0724942824251946, + 1.0 + ], + [ + 0.023995594098924912, + -0.0520840490562838, + 0.008084506765781062, + 0.019706815655492798, + 0.042813182202657354, + -0.1575824828755581, + 1.0 + ], + [ + 0.014543768492913627, + -0.03695187138480016, + 0.012867005916286287, + 0.038642818983397824, + 0.04293401387394253, + -0.09014170121073148, + 1.0 + ], + [ + 0.004032210212267935, + -0.020795778048878947, + 0.013575202364126055, + 0.026438734951094067, + 0.0009910674782602487, + -0.031236371852793474, + 1.0 + ], + [ + -0.001491924189136543, + -0.0015754921484312935, + 0.006450877274523192, + 0.025039345696795633, + 0.0010753897689022769, + 0.007630359991997916, + 1.0 + ], + [ + -0.01117371499304382, + 0.010926631123459487, + 0.02330345227015427, + 0.02176429733153316, + -0.05832501142059404, + 0.02897789437879744, + 1.0 + ], + [ + -0.0022676795666220174, + 0.002016242689569477, + 0.009356943080798768, + 0.01325523002316173, + -0.014104715726007983, + -0.008859447001058706, + 1.0 + ], + [ + 9.005737335189796e-05, + -0.0002050975574721233, + 0.0012395860101222392, + 0.0012754623500119086, + 0.0001648717923144895, + -0.0052991438305382286, + 1.0 + ], + [ + 0.00021844604870801593, + -0.0012336428408185735, + 0.0006263669633550195, + 0.0035481807278284977, + 0.0035854449079563183, + -0.005065706988770248, + 0.0 + ], + [ + -0.0011355234979896457, + -0.0017299561459174877, + 0.0006401788460868138, + 0.013496431530493501, + 0.002035746450532139, + -0.00875542711704406, + 0.0 + ], + [ + 0.006630629506324617, + -0.00022547371378334988, + -0.024243157655715213, + 0.003475918946868751, + 0.056109935046767485, + 0.03383099183128367, + 0.0 + ], + [ + -0.004280586905519036, + 0.023592140321604203, + -0.03622398819551019, + -0.019024214051917027, + 0.06302935696196875, + 0.09552488829571747, + 0.0 + ], + [ + -0.0251523000155999, + 0.0488813539561224, + -0.03942711242351437, + -0.06213077491755416, + 0.03813081575199933, + 0.17788147607408789, + 0.0 + ], + [ + -0.040653261256114265, + 0.0382806719298401, + -0.005805044066809814, + -0.0299081232691564, + -0.04279568408304432, + 0.12885049337583226, + 0.0 + ], + [ + -0.031783467132461524, + 0.054602478118395026, + 0.003639598610925995, + -0.02179186378202462, + -0.016260725376210386, + 0.1492842345022577, + 0.0 + ], + [ + -0.022009664649830635, + 0.029067316437081988, + 0.003288335725863859, + 0.009553649868544845, + -0.011055454249298987, + 0.11032477508913882, + 0.0 + ], + [ + -0.01764931994306746, + 0.02974909147609542, + 0.00561384251366527, + -0.009407308905336021, + -0.012425578965491952, + 0.12236460778874149, + 0.0 + ], + [ + -0.007747303705221667, + 0.0033485176274705196, + 0.006073746373265304, + 0.00016775854488705434, + -0.0129759062985451, + 0.019974295642168807, + 0.0 + ], + [ + -0.010343492510563157, + -0.0003486883788437368, + 0.01164515912444487, + 0.0090606492570695, + -0.04990296641566367, + -0.005793284979126555, + 0.0 + ], + [ + -0.004368391512398469, + 0.0006270354477033025, + 0.004121424654631055, + -0.002473296630663575, + -0.015445752035762162, + 0.00532686749120081, + 0.0 + ], + [ + -0.0012959908503786667, + 0.0016901470440663644, + -0.00029859655229283217, + -0.01088434706627305, + -0.009673461491063982, + 0.012666555571271797, + 0.0 + ], + [ + -0.0009533997828136537, + 0.00392445594457869, + -0.0009675952689407851, + -0.025003917066328072, + -0.002242358092899134, + -0.002637478580524368, + 1.0 + ], + [ + 0.0024840407867635535, + 0.004613215595451541, + -0.018049298067354343, + -0.10007602835658674, + 0.04322166705633294, + -0.08888186104715141, + 1.0 + ], + [ + 0.010519529617247659, + -0.0042968455460748204, + -0.035678172253520594, + -0.07452248300608996, + 0.10106669837924685, + -0.21913811666516367, + 1.0 + ], + [ + 0.013286553269312723, + -0.027135306853575333, + -0.015872839566726173, + 0.05893759372333341, + 0.04652415085084759, + -0.1929719342923542, + 1.0 + ], + [ + 0.006922688347043314, + -0.008491193209320617, + 0.011435156094724017, + 0.043003351180908124, + -0.039773167909471205, + 0.014511612105782445, + 1.0 + ], + [ + 0.0015061250537403291, + -0.002887264367941722, + -0.0009119351404738137, + 0.002382319905468648, + 0.004527876192538199, + -0.007841647884431514, + 1.0 + ] + ], + "state": [ + [ + 0.3678249418735504, + -0.006534198764711618, + 0.1185983419418335, + 3.039159709709235, + -0.16082145273685458, + 0.05029657855629922, + 1.0 + ], + [ + 0.36783701181411743, + -0.006752224173396826, + 0.11862141638994217, + 3.0374210347705564, + -0.1606237292289734, + 0.05251046270132064, + 1.0 + ], + [ + 0.36754974722862244, + -0.006977562792599201, + 0.10584274679422379, + 3.0367556532197675, + -0.125154510140419, + 0.0545710138976574, + 1.0 + ], + [ + 0.3811854422092438, + 0.009948608465492725, + 0.10583815723657608, + 3.008379878597804, + -0.14288172125816348, + 0.13012072443962097, + 1.0 + ], + [ + 0.40028491616249084, + 0.06342776119709015, + 0.09447654336690903, + 3.0064643790298184, + -0.16259479522705078, + 0.2940903604030609, + 1.0 + ], + [ + 0.4066561162471771, + 0.10180381685495377, + 0.07933692634105682, + 3.0299380366974553, + -0.19226831197738647, + 0.3909221589565277, + 1.0 + ], + [ + 0.40580984950065613, + 0.12217164039611816, + 0.06459091603755951, + 3.050370568531104, + -0.18967892229557037, + 0.4226435422897339, + 1.0 + ], + [ + 0.40520042181015015, + 0.12297352403402328, + 0.057859838008880615, + 3.0768529420071324, + -0.1914395242929459, + 0.41500329971313477, + 1.0 + ], + [ + 0.4040921926498413, + 0.10892263799905777, + 0.03359796479344368, + 3.1048181002312383, + -0.13500872254371646, + 0.38206249475479126, + 1.0 + ], + [ + 0.4040452241897583, + 0.10636156797409058, + 0.024100665003061295, + 3.116940891491719, + -0.12058310955762863, + 0.3904574513435364, + 1.0 + ], + [ + 0.40419989824295044, + 0.10661391168832779, + 0.02287626825273037, + 3.1175743007385215, + -0.12061560899019243, + 0.3957978487014771, + 1.0 + ], + [ + 0.4040031135082245, + 0.10785200446844101, + 0.02225150354206562, + 3.120498139532753, + -0.12407675385475157, + 0.40098807215690613, + 0.0 + ], + [ + 0.40237295627593994, + 0.10902480781078339, + 0.021439647302031517, + 3.132898036512085, + -0.1259225457906723, + 0.4098546802997589, + 0.0 + ], + [ + 0.40544024109840393, + 0.1108330562710762, + 0.04632074013352394, + 3.1406222104029773, + -0.18225358426570895, + 0.37600526213645935, + 0.0 + ], + [ + 0.4040642976760864, + 0.08496355265378952, + 0.08119143545627594, + 3.139407327287369, + -0.24452652037143707, + 0.2778017222881317, + 0.0 + ], + [ + 0.38477346301078796, + 0.028721841052174568, + 0.11345992982387543, + 3.121865617232867, + -0.27907443046569824, + 0.09290465712547302, + 0.0 + ], + [ + 0.34760260581970215, + -0.013065462931990623, + 0.10856660455465317, + 3.128750573033937, + -0.23644424974918363, + -0.04038266465067863, + 0.0 + ], + [ + 0.315210223197937, + -0.0664459615945816, + 0.09826505184173584, + -3.1403358088624707, + -0.21941797435283658, + -0.19354590773582458, + 0.0 + ], + [ + 0.2892495393753052, + -0.09097382426261902, + 0.09022924304008484, + -3.106303459154912, + -0.20687225461006165, + -0.3062630891799927, + 0.0 + ], + [ + 0.2651806175708771, + -0.11433675140142441, + 0.0800861269235611, + -3.09054950078065, + -0.18859724700450897, + -0.4303106069564819, + 0.0 + ], + [ + 0.25806134939193726, + -0.11440765857696533, + 0.07250756025314331, + -3.0867155251377305, + -0.1745837926864624, + -0.4498927593231201, + 0.0 + ], + [ + 0.25113359093666077, + -0.10996613651514053, + 0.05927887186408043, + -3.0790806283527097, + -0.12506741285324097, + -0.4413122832775116, + 0.0 + ], + [ + 0.24752561748027802, + -0.10866915434598923, + 0.05461383983492851, + -3.081000981973954, + -0.10931786894798277, + -0.4456893503665924, + 0.0 + ], + [ + 0.24560889601707458, + -0.10964316874742508, + 0.05466698110103607, + -3.090564990537711, + -0.09888719022274017, + -0.45780545473098755, + 0.0 + ], + [ + 0.242935910820961, + -0.11275043338537216, + 0.05533527955412865, + -3.1158415881269654, + -0.09678190201520921, + -0.45504412055015564, + 1.0 + ], + [ + 0.24136982858181, + -0.11763567477464676, + 0.07341596484184265, + 3.0586386789851865, + -0.1418997496366501, + -0.36649963259696966, + 1.0 + ], + [ + 0.249038428068161, + -0.11282596737146378, + 0.10974934697151184, + 2.953255923586436, + -0.22092695534229279, + -0.13496361672878265, + 1.0 + ], + [ + 0.2635881006717682, + -0.08490248769521713, + 0.12291713804006577, + 2.97143009503419, + -0.2263414263725281, + 0.06847899407148361, + 1.0 + ], + [ + 0.2727234363555908, + -0.07782872021198273, + 0.11208689212799072, + 3.0191428308659276, + -0.18954962491989136, + 0.04707021266222, + 1.0 + ], + [ + 0.27395689487457275, + -0.07479028403759003, + 0.11291322112083435, + 3.0199278091364583, + -0.1930791288614273, + 0.055563967674970634, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.04393261671066284, + 0.3038042187690735, + 0.6107503175735474, + 0.8281095325946808, + 0.9156777262687683, + 0.9257014021277428, + 0.9260224476456642, + 0.9260224476456642, + 0.9260224476456642, + 0.9260224476456642, + 0.9260224476456642, + 0.9260224476456642, + 0.9260224476456642, + 0.898399829864502, + 0.7218906283378601, + 0.43697023391723633, + 0.15549510717391968, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "83", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/83/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/9496.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/9496.json new file mode 100644 index 00000000..71a2e794 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/train/9496.json @@ -0,0 +1,489 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "put potato on plate" + ], + "videos": [ + { + "video_path": "videos/train/9496/rgb.mp4" + } + ], + "action": [ + [ + -0.0010446633510046658, + 6.163146408820955e-05, + 0.001008631930561206, + 0.00065186249215139, + -0.004456961453820739, + -0.000592822468346282, + 1.0 + ], + [ + 0.0027615325329129405, + -0.007333494171658885, + 0.02637489267978839, + -0.006227336050037183, + -0.028245087830200383, + 0.04904251391396301, + 1.0 + ], + [ + 0.010712375873458772, + -0.02227361162856293, + 0.026377976399886167, + 0.01738444828281215, + 0.020770146233705468, + 0.003532913316101252, + 1.0 + ], + [ + 0.012668624149821055, + 0.007813530233656201, + 0.021231246263330044, + 0.0005841174632635483, + 0.05145926806383245, + 0.04674906403535143, + 1.0 + ], + [ + 0.001382599693177435, + -0.010308975252574374, + 0.01715358849836208, + -0.000502223059611559, + 0.017103674725040617, + -0.005222710187765315, + 1.0 + ], + [ + -0.0021898356558253057, + -0.0020051116569117836, + 0.017904108081176016, + -0.005514560853227666, + 0.018643806147618486, + 0.0041150485810992335, + 1.0 + ], + [ + -0.0007229274976314989, + -0.001392475852305625, + 0.010341046514735422, + -0.022701901695402628, + 0.022585471948969656, + -0.017334346725832336, + 1.0 + ], + [ + -0.0015888326369725603, + -0.00395453612061848, + 0.0119235639537489, + -0.01821568864336513, + 0.013732304619435158, + -0.0015894690450572045, + 1.0 + ], + [ + 0.0007368252973954613, + 0.0030850196555384426, + 0.009555915016765852, + -0.02621974679473935, + 0.015646077416545515, + -0.013731409861536854, + 1.0 + ], + [ + 0.0038115037226983525, + 0.0038353784809153184, + 0.01048843927882886, + -0.026982178422483204, + 0.020158974097760768, + -0.004443930950526641, + 1.0 + ], + [ + 0.004798827189654864, + 0.0009743239667061076, + 0.008421150871580175, + -0.01900179663594936, + 0.03348860468455951, + 0.0015324202509484882, + 1.0 + ], + [ + 0.002887556790833546, + -0.0016582742752000881, + 0.004995423157298236, + -0.0410518666723067, + 0.017190727037444595, + -0.01108851747019635, + 1.0 + ], + [ + 0.0014542317026421945, + -0.00250983161829034, + -3.9292480006193805e-05, + -0.017057904439995913, + -0.004646871363214347, + -0.01270070474520894, + 0.0 + ], + [ + -0.01238102408623826, + -0.0011797413762413105, + -0.011819178363834042, + -0.017781424103880263, + -0.08971815856431835, + 0.010428737654072413, + 0.0 + ], + [ + 0.006983134244141528, + -0.00802816388231457, + -0.039283505561471144, + 0.02646777889695467, + 0.005872858973497212, + 0.0030967418274513354, + 0.0 + ], + [ + -0.0026053931389818444, + -0.010520517987609319, + -0.03862967830483293, + 0.06576609758345166, + -0.014178916507383024, + 0.013649024834936923, + 0.0 + ], + [ + -0.009986179999449478, + -0.019049661250847706, + -0.02068564194358659, + 0.032293361080885624, + -0.028178790307001506, + 0.04579852315175923, + 0.0 + ], + [ + -0.012491835571127626, + -0.03451651395716958, + -0.007072179072855449, + 0.02365338606120335, + -0.012733900074105119, + 0.017956852160233024, + 0.0 + ], + [ + -0.00647166462462176, + -0.02393472601963849, + -0.0030917358485306624, + 0.007273763577930806, + 0.0031733745372664055, + 0.01685197317426271, + 0.0 + ], + [ + 0.005238880271660455, + -0.018252969906232475, + -0.007532179861946861, + -0.0031540779486172667, + 0.011042251601278577, + 0.00433679839689029, + 0.0 + ], + [ + 0.0021073622719431652, + -0.005184753566134395, + 0.004466870942442457, + -0.020823101722668116, + 0.032802934381690914, + 0.029564924041775673, + 0.0 + ], + [ + -0.0019053223461310013, + -0.0008932649373068207, + 0.006962976203640884, + -0.037653337545801496, + 0.03618204764151709, + 0.03051519932965075, + 0.0 + ], + [ + -0.0023334664120623254, + 0.001203470912749045, + 0.005726217562942328, + -0.024123853262369068, + 5.205308895550321e-05, + 0.0036730051533636287, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.2385996348995914e-17, + 3.140728329586667e-17, + -8.44116425433712e-18, + 1.0 + ] + ], + "state": [ + [ + 0.27595746517181396, + -0.0034280673135071993, + 0.06388033926486969, + -2.943692850070544, + -0.07372388988733293, + 0.011083460412919521, + 1.0 + ], + [ + 0.2749879062175751, + -0.0033009261824190617, + 0.06280504912137985, + -2.943146826820918, + -0.06947039812803268, + 0.012544513680040836, + 1.0 + ], + [ + 0.27928152680397034, + 0.009143183007836342, + 0.038644127547740936, + -2.946523832278796, + -0.03205111995339393, + -0.02996402233839035, + 1.0 + ], + [ + 0.29148203134536743, + 0.03575373440980911, + 0.017438169568777084, + -2.9288618882470807, + -0.05174160376191139, + -0.03746495768427849, + 1.0 + ], + [ + 0.3051644563674927, + 0.032083820551633835, + -0.004278968553990126, + -2.9254014660888394, + -0.09209325909614562, + -0.09421369433403014, + 1.0 + ], + [ + 0.3091599643230438, + 0.04551614820957184, + -0.018633084371685982, + -2.926004262762614, + -0.1099189817905426, + -0.09277338534593582, + 1.0 + ], + [ + 0.30939245223999023, + 0.05130830034613609, + -0.03583104535937309, + -2.930602120357104, + -0.12724731862545013, + -0.10084591060876846, + 1.0 + ], + [ + 0.31027400493621826, + 0.05476441606879234, + -0.045663490891456604, + -2.954834358888217, + -0.15295250713825229, + -0.08848617970943452, + 1.0 + ], + [ + 0.3109155297279358, + 0.060831133276224136, + -0.05675932392477989, + -2.9728796650939664, + -0.1667410582304001, + -0.08948805183172227, + 1.0 + ], + [ + 0.3131536841392517, + 0.059187889099121094, + -0.06643745303153992, + -3.000930504994937, + -0.1844605803489685, + -0.07839231193065643, + 1.0 + ], + [ + 0.3187040090560913, + 0.056417692452669144, + -0.07647562772035599, + -3.0281788428598126, + -0.20504322648048398, + -0.0767855942249298, + 1.0 + ], + [ + 0.32510828971862793, + 0.055909886956214905, + -0.08379827439785004, + -3.0460049753361425, + -0.23814022541046137, + -0.08225080370903014, + 1.0 + ], + [ + 0.3292115032672882, + 0.0577063262462616, + -0.08779559284448624, + -3.0893384647839746, + -0.256300002336502, + -0.07253798097372055, + 1.0 + ], + [ + 0.330752968788147, + 0.06010527163743973, + -0.08726217597723007, + -3.1097840165072164, + -0.25230094790458685, + -0.059189390391111374, + 0.0 + ], + [ + 0.31587928533554077, + 0.06179133057594299, + -0.07887735217809677, + -3.1255248860740146, + -0.16228942573070526, + -0.06682159751653671, + 0.0 + ], + [ + 0.3168959617614746, + 0.06913579255342484, + -0.038859423249959946, + -3.0985339937084397, + -0.16811093688011172, + -0.07005786895751953, + 0.0 + ], + [ + 0.30843567848205566, + 0.07859916985034943, + -0.0007990828016772866, + -3.0305618812614163, + -0.1533432900905609, + -0.08323715627193451, + 0.0 + ], + [ + 0.2965337932109833, + 0.09629029035568237, + 0.020078621804714203, + -2.9918035884671887, + -0.12012669444084169, + -0.12591826915740967, + 0.0 + ], + [ + 0.2869400978088379, + 0.13084332644939423, + 0.030637983232736588, + -2.966246413188525, + -0.10484065115451814, + -0.14185963571071625, + 0.0 + ], + [ + 0.28307652473449707, + 0.15465715527534485, + 0.03714080899953842, + -2.957193525629588, + -0.10501023381948471, + -0.15910175442695618, + 0.0 + ], + [ + 0.28973206877708435, + 0.17036353051662445, + 0.0483819842338562, + -2.959676446514674, + -0.11506801843643188, + -0.16543127596378326, + 0.0 + ], + [ + 0.293161004781723, + 0.17578013241291046, + 0.04519162327051163, + -2.9764536191993436, + -0.14191101491451266, + -0.2007846087217331, + 0.0 + ], + [ + 0.2926482856273651, + 0.17795176804065704, + 0.038268305361270905, + -3.008932115631648, + -0.17249231040477753, + -0.2373577207326889, + 0.0 + ], + [ + 0.2912849485874176, + 0.17783354222774506, + 0.03211929649114609, + -3.032421404617377, + -0.17205691337585452, + -0.24106000363826754, + 1.0 + ], + [ + 0.2912849485874176, + 0.17783354222774506, + 0.03211929649114609, + -3.032421404617377, + -0.17205691337585452, + -0.24106000363826754, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.17852425575256348, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.33987146615982056, + 0.1303747296333313, + 0.0, + 0.0 + ], + "episode_id": "9496", + "latent_videos": [ + { + "latent_video_path": "latent_videos/train/9496/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/1527.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/1527.json new file mode 100644 index 00000000..cab54e82 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/1527.json @@ -0,0 +1,584 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "PLACE THE BRUSH ON THE ORANGE CLOTH" + ], + "videos": [ + { + "video_path": "videos/val/1527/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + 4.46478253197146e-10, + -4.688638490197737e-09, + -0.00153398564292649, + 1.0 + ], + [ + -0.0015969392276188913, + -0.00020677812550743844, + 0.008850616303854537, + -0.012851051952218306, + -0.022703812833452807, + -0.0018963113809367747, + 1.0 + ], + [ + -0.005254556658652358, + -0.04800761897796832, + 0.005345644940651196, + 0.0018273177736492846, + 0.042935672650673704, + -0.13250041449757408, + 1.0 + ], + [ + 0.008929979632812784, + -0.027672661656533624, + 0.006094748105208993, + 0.021735333406806678, + 0.004192799043866376, + -0.058099377163574654, + 1.0 + ], + [ + 0.008354544036850804, + -0.03581066297859499, + 0.00486961829424437, + 0.058763100722897595, + -3.291472700306768e-05, + -0.07328447278822167, + 1.0 + ], + [ + 0.014340719163114288, + -0.019367321268581227, + 0.017309277687930693, + 0.050901482624231144, + -0.01882263134384541, + -0.05411853310435643, + 1.0 + ], + [ + 0.010390368434383212, + -0.00983811123775331, + 0.008894646624588786, + 0.039895158366800035, + -0.017265758161432453, + -0.016537594440144753, + 1.0 + ], + [ + 0.003225302422792462, + -0.009526589774630664, + 0.016398962085764204, + 0.008318894987838837, + -0.018329936881455504, + -0.036469079276197934, + 1.0 + ], + [ + 0.005535544750322436, + -0.006860415627922475, + 0.01198516894402459, + 0.014157888809372712, + -0.01385770002043199, + -0.02607952389636573, + 1.0 + ], + [ + 0.000952350775067972, + 0.0013233278246136868, + 0.0015230790724726651, + 0.015106188878465494, + 0.006272071213782807, + 0.0053108509372450366, + 1.0 + ], + [ + -0.004077177597056107, + 0.005646880190346535, + 0.0014886662083822438, + -0.000809922023707952, + 0.010649384983415706, + 0.00986709589000703, + 1.0 + ], + [ + -0.00164920883967962, + 0.004070950024726741, + -0.0018022002015604256, + -0.0047516035929665665, + 0.006158437317513167, + 0.014930432069900862, + 0.0 + ], + [ + -0.0024917752962640566, + 0.0029379697961877116, + -0.0007305325001621138, + -0.023878576469302974, + -0.005782805628630972, + -0.007222587872823425, + 0.0 + ], + [ + -0.0024487090335059045, + -0.0048797121235225685, + -0.02618629677574524, + -0.03251851027847331, + 0.024463645566731952, + -0.027686229539276977, + 0.0 + ], + [ + 0.004406036034761031, + 0.010175452102412144, + -0.04825448303781464, + -0.0016070561284189518, + 0.09772153273054039, + 0.0031283446564027233, + 0.0 + ], + [ + -0.006246249112819696, + 0.04002483670761027, + -0.021772467690721532, + -0.0503533332280843, + 0.022247678133297713, + 0.08156829114843112, + 0.0 + ], + [ + -0.0037980895863288506, + 0.04525586681001987, + 0.0004662311804277554, + -0.04638211316953231, + -0.013780743456301836, + 0.09990008698591585, + 0.0 + ], + [ + 0.0006394997034945078, + 0.03333706609527571, + 0.014800972743186282, + -0.0019005685760822581, + -0.04049601926114914, + 0.0675981589457608, + 0.0 + ], + [ + 0.004529021118242808, + 0.010686161196145199, + 0.023853036002367232, + 0.024242304826434993, + -0.05848975635386035, + 0.02659574419318436, + 0.0 + ], + [ + 0.013843843846784856, + 0.005675913154374833, + 0.0126447350149563, + 0.03180506118398647, + -0.021060957595347762, + 0.026864358061581925, + 0.0 + ], + [ + 0.014797182174824865, + 0.0055386685513799105, + 0.006299858916760252, + 0.01880041692388678, + 0.004565666382462498, + 0.01738965836824407, + 0.0 + ], + [ + 0.005658306473038631, + 0.009517627365259034, + 0.004224968485502898, + 0.016803624336994737, + -0.020160856748982017, + 0.033918684755893894, + 0.0 + ], + [ + 0.0011024983428922271, + 0.002297626883108057, + 0.002767122304427486, + 0.010803853676675567, + -0.014939548946189672, + 0.012845773501814652, + 0.0 + ], + [ + -0.0008309861192838533, + -0.0010409210154153053, + -0.0002577377871299065, + 0.0065398304931866815, + -0.00346715870066558, + 0.015927866991021496, + 1.0 + ], + [ + -0.010065410853824197, + 0.002445067350961525, + -0.0016397346723890183, + -0.013524961349409921, + -0.005072267481152233, + 0.02127758279525929, + 1.0 + ], + [ + -0.01807795152937108, + -0.010466059089340184, + -0.033312618057478635, + -0.047854598513752886, + 0.08062274727586667, + -0.045698954265299456, + 1.0 + ], + [ + -0.015795960438287137, + -0.012587639205582458, + 0.000613710470945034, + -0.05074418586675819, + -0.014300166091103338, + -0.03137564064982897, + 1.0 + ], + [ + -0.010565509765680445, + -0.001291140635937436, + 0.008804083618519616, + -0.0015521377541387578, + -0.02611143091317778, + -0.003878444399313224, + 1.0 + ], + [ + 0.00017298263836461493, + -0.003227722653940275, + -0.0041635447132083525, + 0.006657751625549457, + 0.011298644432476152, + -0.0034530564304923856, + 1.0 + ] + ], + "state": [ + [ + 0.289588987827301, + 0.0359116867184639, + 0.09272906929254532, + 3.0821795148127755, + -0.09576564282178879, + -0.11120239645242691, + 1.0 + ], + [ + 0.289588987827301, + 0.0359116867184639, + 0.09272906929254532, + 3.0820324925058564, + -0.09567444026470184, + -0.10966408252716064, + 1.0 + ], + [ + 0.28881391882896423, + 0.03567466884851456, + 0.08376974612474442, + 3.0691142325573644, + -0.07289800047874449, + -0.10912150889635086, + 1.0 + ], + [ + 0.28941458463668823, + 0.08338751643896103, + 0.07460235059261322, + 3.061663122969218, + -0.10548736900091171, + 0.026802873238921165, + 1.0 + ], + [ + 0.29843753576278687, + 0.11073683202266693, + 0.0673038586974144, + 3.077370086806365, + -0.10485094785690308, + 0.0853734016418457, + 1.0 + ], + [ + 0.3044417202472687, + 0.14680448174476624, + 0.06105969473719597, + 3.128619661676236, + -0.09983786195516586, + 0.15887105464935303, + 1.0 + ], + [ + 0.3172312378883362, + 0.16823890805244446, + 0.045017436146736145, + -3.109031709032603, + -0.08017006516456604, + 0.21290463209152222, + 1.0 + ], + [ + 0.3258863091468811, + 0.18046626448631287, + 0.037307415157556534, + -3.0705078562074384, + -0.06344009935855865, + 0.23002758622169495, + 1.0 + ], + [ + 0.3275561034679413, + 0.191813126206398, + 0.021862514317035675, + -3.0646184032135686, + -0.047700997442007065, + 0.2677440345287323, + 1.0 + ], + [ + 0.3313601613044739, + 0.2009052038192749, + 0.010717331431806087, + -3.0517727007442197, + -0.03587207198143005, + 0.2948272228240967, + 1.0 + ], + [ + 0.3326702415943146, + 0.200068399310112, + 0.00911689829081297, + -3.0364562218361577, + -0.04164186865091324, + 0.28897020220756525, + 1.0 + ], + [ + 0.33040398359298706, + 0.19369903206825256, + 0.006875917315483093, + -3.03681085457141, + -0.05119450017809868, + 0.27802619338035583, + 1.0 + ], + [ + 0.329915851354599, + 0.18915338814258575, + 0.008156267926096916, + -3.0407790934019765, + -0.055751498788595207, + 0.26250964403152466, + 0.0 + ], + [ + 0.3282676041126251, + 0.18560761213302612, + 0.008447887375950813, + -3.0650924315028867, + -0.05072326585650444, + 0.2702874541282653, + 0.0 + ], + [ + 0.3238521218299866, + 0.18735626339912415, + 0.03477231785655022, + -3.0989219938688954, + -0.07721423357725143, + 0.29609119892120367, + 0.0 + ], + [ + 0.32809582352638245, + 0.17587006092071533, + 0.08274633437395096, + -3.099753139662095, + -0.17471100389957428, + 0.2887061834335327, + 0.0 + ], + [ + 0.3304983079433441, + 0.13391728699207306, + 0.10143420100212097, + -3.135656066360422, + -0.19293011724948883, + 0.20473450422286987, + 0.0 + ], + [ + 0.33618661761283875, + 0.08888087421655655, + 0.09998472034931183, + 3.1205313374572476, + -0.1775879263877869, + 0.1033276990056038, + 0.0 + ], + [ + 0.3427594006061554, + 0.055740684270858765, + 0.08622375875711441, + 3.1308591897883495, + -0.13810619711875918, + 0.03428706899285317, + 0.0 + ], + [ + 0.3508836328983307, + 0.045071348547935486, + 0.06333630532026291, + -3.124339036765047, + -0.07985415309667587, + 0.00702379085123539, + 0.0 + ], + [ + 0.365737646818161, + 0.0397186353802681, + 0.05174044147133827, + -3.0904192013018807, + -0.05830485746264458, + -0.019512249156832695, + 0.0 + ], + [ + 0.38078829646110535, + 0.03421476110816002, + 0.046038951724767685, + -3.070598656433173, + -0.06196605041623116, + -0.03714638203382492, + 0.0 + ], + [ + 0.3863929808139801, + 0.02480616793036461, + 0.04150928184390068, + -3.0518100579553327, + -0.03941786289215088, + -0.06956712156534195, + 0.0 + ], + [ + 0.38746657967567444, + 0.022686146199703217, + 0.03859304264187813, + -3.0405505542927465, + -0.023384150117635723, + -0.08102329820394515, + 0.0 + ], + [ + 0.38671183586120605, + 0.023760369047522545, + 0.03893493488430977, + -3.033659892278262, + -0.018325282260775566, + -0.09652265161275864, + 1.0 + ], + [ + 0.37641873955726624, + 0.022137312218546867, + 0.040117066353559494, + -3.0468288530879697, + -0.010986706241965294, + -0.11713085323572159, + 1.0 + ], + [ + 0.35894232988357544, + 0.03151088207960129, + 0.07406986504793167, + -3.094859706359454, + -0.09556232392787933, + -0.07923901081085206, + 1.0 + ], + [ + 0.34426772594451904, + 0.0453183576464653, + 0.07253777980804443, + 3.1344980682530483, + -0.08269421756267549, + -0.04712269827723504, + 1.0 + ], + [ + 0.33453503251075745, + 0.04700734093785286, + 0.06288216263055801, + 3.132638025219091, + -0.0565553642809391, + -0.04342499002814293, + 1.0 + ], + [ + 0.3346158266067505, + 0.050271786749362946, + 0.06701980531215668, + 3.139093903247737, + -0.06782227754592896, + -0.03986293077468872, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0011776089668273926, + 0.05079597234725952, + 0.3308374881744385, + 0.6296481192111969, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.6692290902137756, + 0.5949552357196808, + 0.2978169918060303, + 0.02981734275817871, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "1527", + "latent_videos": [ + { + "latent_video_path": "latent_videos/val/1527/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/199.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/199.json new file mode 100644 index 00000000..f87c0570 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/199.json @@ -0,0 +1,622 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Move the white item to sit between blue cloth and spoon" + ], + "videos": [ + { + "video_path": "videos/val/199/rgb.mp4" + } + ], + "action": [ + [ + 0.0, + 0.0, + 0.0, + 4.089367193370084e-10, + 8.443038961778119e-09, + -0.001533983066826248, + 1.0 + ], + [ + -0.0058122712542445585, + 0.004294748823452637, + 0.018653688099710627, + -0.02409099730884047, + -0.04474758892795706, + 0.004256768028888944, + 1.0 + ], + [ + -0.010145943830697817, + 0.03394877795118116, + 0.008297199068437122, + -0.030626709126104372, + -0.026908209112141455, + 0.0844998028625207, + 1.0 + ], + [ + -0.004205655223294792, + 0.04040185710927649, + -0.01264738811271286, + 0.012819547885140805, + 0.007545302401201341, + 0.09277351999188982, + 1.0 + ], + [ + -0.014599884426573671, + 0.029027597221568145, + -0.006034655802780252, + -0.032810262953068756, + -0.0004960047628343351, + 0.06749599994674887, + 1.0 + ], + [ + -0.004599887425123325, + 0.04453785150253474, + 0.0009310806606805907, + -0.03221603422867791, + 0.0008793496263805207, + 0.12664517170936568, + 1.0 + ], + [ + -0.0070251030288180855, + 0.021302002153526442, + 0.003490106638909116, + 0.0011925201901885208, + -0.022253167168811983, + 0.05361323320100611, + 1.0 + ], + [ + -0.0005842017693121341, + 0.018814804820994933, + 0.00843230691320725, + -0.020598802972321903, + -0.015927977290303246, + 0.04506933907040478, + 1.0 + ], + [ + -0.001210998547276948, + 0.0005481680744507707, + 0.006935238662987605, + -0.005527430619122432, + -0.02455570999071266, + -0.004816577960250219, + 1.0 + ], + [ + -0.0006665046615948918, + 0.0012107724356064857, + 0.006336575345263255, + -0.01062596421690094, + -0.01707931390201515, + -0.00473627988357506, + 1.0 + ], + [ + -0.002594918056806819, + 0.0010781604102387881, + 0.007412944929195274, + -0.011890081293896742, + -0.02660857204721755, + -0.004755874413066639, + 1.0 + ], + [ + -0.0004503564565023925, + 0.003128985992912312, + 0.0067861257193173406, + -0.007087799613554346, + -0.023856844185920115, + 0.005946692827519825, + 1.0 + ], + [ + 0.0009999111769106092, + 0.0028921625843928113, + 0.00659072553182592, + -0.02612476229953334, + -0.0032136858902915193, + 0.00035120081894772043, + 1.0 + ], + [ + 0.0006999236243060205, + -0.000700005762939483, + 0.0009605662035535195, + -0.016967205845740553, + 0.015505640280187463, + -0.00047845006013068174, + 1.0 + ], + [ + -0.00247014707470403, + -0.000927277804756997, + 0.0007268721751016307, + -0.005338992030398866, + -0.0037427401855249613, + -0.0017746579986171462, + 0.0 + ], + [ + -0.0031451390361429098, + 0.00068941359903711, + -0.005070949814182911, + -0.002899757895520684, + -0.006984213151739086, + -0.005946455086147693, + 0.0 + ], + [ + -0.0019764660148765436, + -0.007580938517768371, + -0.03063716065017813, + -0.006850569784313236, + 0.06015598424573097, + -0.05758323139643647, + 0.0 + ], + [ + 0.018062239205547146, + -0.014742043785507931, + -0.03590043075883523, + 0.002977820218997576, + 0.04951592123593558, + -0.07646877102614213, + 0.0 + ], + [ + 0.032814851573024266, + -0.01595785432340375, + 0.002551128162776729, + 0.0025659076938508232, + 0.017862408097246603, + -0.032201324404448804, + 0.0 + ], + [ + 0.021840493360800448, + -0.009406108172969482, + 0.011055849335196186, + 0.006174348455719754, + 0.0006844792791273347, + -0.007943735701722392, + 0.0 + ], + [ + 0.010329423902797013, + -0.016619483221766655, + 0.00939863932014306, + 0.07172798863366281, + -0.011546345513273389, + -0.005840662503878794, + 0.0 + ], + [ + 0.01175718588650213, + -0.013806766592008418, + 0.002632532878185932, + 0.03797896260839082, + 0.025136630800438747, + 0.020581837092666, + 0.0 + ], + [ + -0.002296499301157879, + -0.017148159351601314, + 0.002147513338143422, + 0.012975825959247194, + -0.002143434675226681, + -0.0070183210924843874, + 0.0 + ], + [ + -0.001523755941161498, + 0.0006926215057545661, + 0.004127589092384302, + -0.0062038709528615885, + -0.0140281470341169, + 0.024742171473270437, + 0.0 + ], + [ + -0.0017698193617940784, + 0.003818693282388121, + 0.004296582186321722, + 0.013212538132561364, + -0.013959002679038834, + 0.01946072362531635, + 1.0 + ], + [ + -0.003687169037730824, + 0.002136068465532415, + 0.006652096113108225, + 0.01238073357172233, + -0.01645704189018183, + -0.005572249829597233, + 1.0 + ], + [ + -0.004426094981679318, + -0.000146988551670702, + -0.0020635998144884265, + 0.01768372552902754, + 0.0031430096622518493, + -0.016129581687324104, + 1.0 + ], + [ + -0.003472326297469028, + 0.0004346732611364774, + -0.015562168267138314, + 0.015839335943909228, + 0.04315426386380142, + -0.027158067569254613, + 1.0 + ], + [ + -0.0030935787531547573, + 0.0014288391741287716, + 0.007127975007158966, + 0.002939393789028113, + 0.004047529133778957, + -0.06240877123801388, + 1.0 + ], + [ + -0.0033436567290959897, + -0.003932399382532507, + -0.00987356282806081, + 0.040605173169121773, + 0.054114585427340434, + -0.055935417640117314, + 1.0 + ], + [ + 0.000995823044834454, + -0.0005262733121732777, + -0.0031840310152863025, + 0.002781450093706607, + 0.01191413244723356, + -0.005299036410256499, + 1.0 + ] + ], + "state": [ + [ + 0.3929700553417206, + 0.1324481964111328, + 0.10977274924516678, + -3.052886875467845, + -0.15580102801322937, + -0.09183102846145631, + 1.0 + ], + [ + 0.3929700553417206, + 0.1324481964111328, + 0.10977274924516678, + -3.053126985328742, + -0.1559367477893829, + -0.0902843102812767, + 1.0 + ], + [ + 0.38994720578193665, + 0.1300812065601349, + 0.09013994038105011, + -3.0770847295695027, + -0.11098816990852355, + -0.09056978672742844, + 1.0 + ], + [ + 0.3780439794063568, + 0.09768170863389969, + 0.07861219346523285, + -3.0987421741061887, + -0.07831631600856782, + -0.17337678372859958, + 1.0 + ], + [ + 0.3660169243812561, + 0.05825890600681305, + 0.08915480226278305, + -3.0788161252909383, + -0.08154696971178055, + -0.266697883605957, + 1.0 + ], + [ + 0.3439149856567383, + 0.03387267142534256, + 0.09215329587459564, + -3.106272129463502, + -0.07663563638925554, + -0.3342280387878418, + 1.0 + ], + [ + 0.32517313957214355, + -0.006701675709336996, + 0.08930519968271255, + -3.12907881767536, + -0.07243981212377548, + -0.4611597359180451, + 1.0 + ], + [ + 0.3096829354763031, + -0.02274090237915516, + 0.08505020290613174, + -3.1240381953590592, + -0.04941481351852417, + -0.5145418643951416, + 1.0 + ], + [ + 0.3003665506839752, + -0.03891398012638092, + 0.07627076655626297, + 3.1407454201304397, + -0.03264888375997544, + -0.5593425631523132, + 1.0 + ], + [ + 0.29923853278160095, + -0.038861460983753204, + 0.06930015981197357, + 3.1350611905851444, + -0.008088726550340652, + -0.5545480847358705, + 1.0 + ], + [ + 0.29805612564086914, + -0.03960195928812027, + 0.06296644359827042, + 3.1243969352268657, + 0.009021244011819363, + -0.5499239563941954, + 1.0 + ], + [ + 0.2951570153236389, + -0.03923873230814934, + 0.05559684336185455, + 3.112539716186472, + 0.03570757433772087, + -0.5456252098083496, + 1.0 + ], + [ + 0.29284265637397766, + -0.04172322526574135, + 0.04892481863498688, + 3.1052069162302693, + 0.05938081815838814, + -0.552272379398346, + 1.0 + ], + [ + 0.2917228937149048, + -0.04470977187156677, + 0.04239574819803238, + 3.079054148989268, + 0.06257959455251692, + -0.5527411699295044, + 1.0 + ], + [ + 0.2925993502140045, + -0.04450010508298874, + 0.041351500898599625, + 3.0621700530224523, + 0.04713410511612892, + -0.5512930750846863, + 1.0 + ], + [ + 0.2909194231033325, + -0.042449723929166794, + 0.040670622140169144, + 3.0569000412994107, + 0.0510057955980301, + -0.5498190522193909, + 0.0 + ], + [ + 0.2883286774158478, + -0.041164547204971313, + 0.04593541845679283, + 3.054271848993846, + 0.05846726521849632, + -0.5444757342338562, + 0.0 + ], + [ + 0.29342907667160034, + -0.03230077028274536, + 0.0758591890335083, + 3.05109045852954, + 0.0034446916542947297, + -0.481963187456131, + 0.0 + ], + [ + 0.31784796714782715, + -0.02484358660876751, + 0.11021790653467178, + 3.054529668884822, + -0.038976382464170456, + -0.40135303139686584, + 0.0 + ], + [ + 0.354297012090683, + -0.023283353075385094, + 0.10757052153348923, + 3.055818835394927, + -0.053949613124132156, + -0.36767587065696716, + 0.0 + ], + [ + 0.37827110290527344, + -0.023490220308303833, + 0.0969444215297699, + 3.0615653340988835, + -0.05394933745265007, + -0.35969108343124395, + 0.0 + ], + [ + 0.39403197169303894, + -0.012520887888967991, + 0.08681982755661011, + 3.1330245427140078, + -0.041972387582063675, + -0.3547882139682769, + 0.0 + ], + [ + 0.4099437892436981, + -0.003716480452567339, + 0.08456484973430634, + -3.1113263071798762, + -0.06727571040391922, + -0.3751935660839081, + 0.0 + ], + [ + 0.41421860456466675, + 0.013091514818370342, + 0.08278655260801315, + -3.0988281351798257, + -0.0653439536690712, + -0.3680984675884246, + 0.0 + ], + [ + 0.4128674864768982, + 0.013060069642961025, + 0.07854249328374863, + -3.1034634319418153, + -0.050251953303813934, + -0.39224618673324585, + 0.0 + ], + [ + 0.4100441038608551, + 0.010275756940245628, + 0.07402016967535019, + -3.0893033650987825, + -0.03555230796337128, + -0.41117063164711, + 1.0 + ], + [ + 0.40617284178733826, + 0.010015901178121567, + 0.06713873147964478, + -3.0771450345688542, + -0.019408261403441426, + -0.4047454297542572, + 1.0 + ], + [ + 0.4020736515522003, + 0.011786979623138905, + 0.06912122666835785, + -3.0597776641421994, + -0.023581044748425484, + -0.38884744048118586, + 1.0 + ], + [ + 0.39787739515304565, + 0.011663648299872875, + 0.08450964093208313, + -3.044440949457236, + -0.06880360841751099, + -0.36527642607688904, + 1.0 + ], + [ + 0.39519819617271423, + 0.011905970051884651, + 0.0770811140537262, + -3.0459402893954, + -0.07875049859285355, + -0.30336406826972967, + 1.0 + ], + [ + 0.39213868975639343, + 0.01597728580236435, + 0.08699044585227966, + -3.009327234821864, + -0.13785648345947263, + -0.25245586037635803, + 1.0 + ], + [ + 0.3926900029182434, + 0.015940167009830475, + 0.09032231569290161, + -3.0070489217811307, + -0.1503644436597824, + -0.24873268604278564, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.010668635368347168, + 0.07526767253875732, + 0.36297857761383057, + 0.6447625160217285, + 0.8033039718866348, + 0.8576638251543045, + 0.879532054066658, + 0.8854858502745628, + 0.8859040141105652, + 0.8859040141105652, + 0.886321172118187, + 0.8614679723978043, + 0.634721577167511, + 0.3268251419067383, + 0.04696917533874512, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "199", + "latent_videos": [ + { + "latent_video_path": "latent_videos/val/199/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/1996.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/1996.json new file mode 100644 index 00000000..6a721de9 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/1996.json @@ -0,0 +1,641 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "PICK UP THE VESSEL AND PUT ANOTHER SIDE OF THE TABLE" + ], + "videos": [ + { + "video_path": "videos/val/1996/rgb.mp4" + } + ], + "action": [ + [ + -4.963611772265726e-05, + -1.7425473994587463e-05, + 0.00039490853600948454, + 0.0005934185257321023, + -0.0014144939464187977, + 1.1745381625430938e-05, + 1.0 + ], + [ + -0.0032832037452427175, + -0.0024645314652085946, + 0.005732075269786685, + 0.015337994531293342, + -0.020214430133554836, + 0.011683175069342215, + 1.0 + ], + [ + -0.011588904454411209, + -0.023556069016972152, + 0.009082295198125511, + 0.07442401577253087, + -0.05851779400221417, + 0.054586133036733445, + 1.0 + ], + [ + -0.01318621092496055, + -0.02802440928129219, + 0.0017760161303220143, + 0.020527569691461592, + -0.02022591781419146, + 0.003139352095712118, + 1.0 + ], + [ + -0.023992106692449954, + -0.02952126159414816, + -0.0013399926505623305, + 0.039101023848245056, + -0.021042694354506484, + 0.02585061513917338, + 1.0 + ], + [ + -0.007050728073156213, + -0.027756133925268087, + 0.009436039864640177, + -0.008894292665404153, + 0.02038518273352791, + 0.0661585920913077, + 1.0 + ], + [ + 0.012890592170095633, + -0.019657901571573974, + 0.017077258939196104, + 0.031665306408685825, + 0.03400561307690191, + 0.08789753874278149, + 1.0 + ], + [ + 0.0037594648776107568, + -0.018778346632634592, + 0.007853468009944752, + 0.0084271824782549, + 0.059015656385003175, + 0.04938807722402522, + 1.0 + ], + [ + 0.004734333263314315, + -0.012031338978446045, + 0.002989920985851475, + -0.012772179281465403, + 0.021984815869970985, + 0.04638944550694433, + 1.0 + ], + [ + 0.011159733133924347, + -0.009322771792250297, + 0.0023647284362390457, + -0.03219417154573395, + 0.04859428078240033, + 0.043528859159695635, + 1.0 + ], + [ + 0.013541561567205412, + -0.005315900545437498, + 0.0022771445452087024, + -0.009734497352711168, + 0.01321302325486266, + 0.022866269342390264, + 1.0 + ], + [ + 0.01796385279846835, + -0.0003028226948280225, + 0.005501178971764134, + -0.015810896713017115, + 0.018315612803655047, + 0.05746174807884929, + 1.0 + ], + [ + 0.016382254862537745, + 0.0042930985646548415, + -0.0014654596142402603, + 0.01042095360992949, + 0.015131784328477374, + 0.03434396357507916, + 1.0 + ], + [ + 0.02132555158405855, + 0.007906893752930967, + -0.004385192962480611, + 0.015650208717427495, + 0.01475268792668615, + 0.02828203946851487, + 1.0 + ], + [ + 0.02378151845220941, + 0.0035415393179553025, + -0.0032898021121202216, + 0.003794068768283655, + 0.013561947210452523, + -0.009822177874909657, + 1.0 + ], + [ + 0.017298329932573048, + -0.0028783960195233236, + 0.00021786682259599695, + -0.0006437823119920698, + 0.02390988381187062, + -0.03364372453381953, + 1.0 + ], + [ + 2.7226036964792853e-05, + -0.0009430099231788871, + 0.004072985470671461, + 0.006000886068127929, + -0.01322061330261397, + -0.017246269094298914, + 1.0 + ], + [ + 0.018226035973341395, + 0.003399403477756024, + -0.0022895693184246794, + 0.006117137647570378, + 0.011329268595483942, + 0.013704756450626536, + 1.0 + ], + [ + 0.019261872255994805, + 0.005990597786473419, + -0.00026520553307380837, + -0.010553958026227472, + 0.01208326021974439, + 0.0035316605615475973, + 1.0 + ], + [ + 0.011975021512903637, + 0.004729094285480609, + 0.0037465366409188978, + 0.008795437831958169, + -0.009852993275756143, + 0.013670603618794557, + 1.0 + ], + [ + 0.00738124308017023, + 0.007663898178484421, + 0.0024646953928846326, + -0.0011866553567933525, + -0.002134485083936234, + 0.020257895293986583, + 1.0 + ], + [ + 0.008159003059296096, + 0.0029756117225677348, + -0.0018505350373454457, + -0.01075713010263108, + 0.014716188208562683, + -0.00373321323464021, + 1.0 + ], + [ + 0.00071240479504316, + 0.003148256057290745, + -0.0003320346499075385, + -0.020630688333963816, + 0.0026125579112616574, + -0.025626307847800157, + 1.0 + ], + [ + -0.0073251584439368855, + -0.0007027709330253758, + -0.005439048196899402, + 0.01353901738604813, + -0.023283193873892622, + -0.05907583512007154, + 1.0 + ], + [ + -0.010658131279345687, + -0.004835023928584698, + -0.019854209188434894, + 0.006869184840556393, + -0.007548852747331401, + -0.06438501288699416, + 1.0 + ], + [ + -0.011130279775006455, + -0.015321190673124051, + -0.0008238854142552858, + -0.004149923843245763, + -0.024546886708098896, + -0.04502117148628913, + 1.0 + ], + [ + -0.01110725712718325, + -0.013555964611178386, + -0.0003156053209494741, + -0.02692404244984195, + -0.042854089103673464, + -0.09418063495904803, + 1.0 + ], + [ + -0.006122248292784106, + -0.02913723061420666, + -0.005232129022703751, + -0.05156808534535112, + 0.0031143404460436913, + -0.17956797222826143, + 1.0 + ], + [ + 0.004312082023273025, + -0.012055344584242454, + -0.010067774461474633, + -0.017413268817825657, + 0.013196609121230165, + -0.1013506813222962, + 1.0 + ], + [ + 0.0008498205505199905, + -0.02747286026302189, + -0.0026607654887984672, + 0.022689446182799798, + 0.010984976301565232, + -0.11121325496702658, + 1.0 + ], + [ + 0.0007618533266439399, + -0.015253054601326203, + 0.0023513085429837325, + 0.00324924065751858, + 0.0015973696263586199, + -0.0865439731300317, + 1.0 + ], + [ + 0.003601410043994742, + -0.012483786574181886, + -0.00517019728750539, + -0.019833966459618522, + 0.013820678489222759, + -0.05081775883343946, + 1.0 + ] + ], + "state": [ + [ + 0.24267952144145966, + -0.08886487782001495, + 0.08492150902748108, + 3.136431582859256, + -0.032789453864097595, + 0.04156072437763214, + 1.0 + ], + [ + 0.24264225363731384, + -0.08885102719068527, + 0.08452510088682175, + 3.1370256209098777, + -0.031375039368867874, + 0.0415416695177555, + 1.0 + ], + [ + 0.23944224417209625, + -0.08654360473155975, + 0.07868166267871857, + -3.130452901246496, + -0.01121200155466795, + 0.029767943546175953, + 1.0 + ], + [ + 0.2272539585828781, + -0.06324026733636856, + 0.06973295658826828, + -3.055420944588729, + 0.047926221042871475, + -0.02413078211247921, + 1.0 + ], + [ + 0.21479485929012299, + -0.034858107566833496, + 0.07100643962621689, + -3.0349423756175717, + 0.06834722310304642, + -0.025520458817481998, + 1.0 + ], + [ + 0.19191746413707733, + -0.005053734872490168, + 0.07710937410593033, + -2.9974562396579465, + 0.0920029878616333, + -0.049077652394771576, + 1.0 + ], + [ + 0.18582798540592194, + 0.024103794246912003, + 0.07242842018604279, + -3.012959437566348, + 0.08110885322093962, + -0.11769597977399825, + 1.0 + ], + [ + 0.19997401535511017, + 0.044268347322940826, + 0.05701693519949913, + -2.9891782869868955, + 0.0583035908639431, + -0.20934993028640747, + 1.0 + ], + [ + 0.2074703574180603, + 0.06286957114934921, + 0.05189481005072594, + -2.9841045906120023, + 0.007369191385805607, + -0.2670491039752961, + 1.0 + ], + [ + 0.21528853476047516, + 0.07353591918945312, + 0.050794001668691635, + -2.997393937902995, + -0.007079985458403825, + -0.31630587577819824, + 1.0 + ], + [ + 0.22887663543224335, + 0.07915375381708145, + 0.04987252876162529, + -3.029243918257304, + -0.04890861362218857, + -0.3663795292377472, + 1.0 + ], + [ + 0.24356421828269958, + 0.07944941520690918, + 0.048869747668504715, + -3.037816362577029, + -0.059460658580064774, + -0.39062419533729553, + 1.0 + ], + [ + 0.2607761323451996, + 0.07330383360385895, + 0.04450666159391403, + -3.05027540971572, + -0.0716250017285347, + -0.4498192369937897, + 1.0 + ], + [ + 0.27350515127182007, + 0.06226199492812157, + 0.04674416407942772, + -3.0373473187261304, + -0.08351713418960573, + -0.4855212867259979, + 1.0 + ], + [ + 0.28815650939941406, + 0.045123737305402756, + 0.0520494319498539, + -3.019246356683322, + -0.09521038830280304, + -0.5153173804283142, + 1.0 + ], + [ + 0.3065914809703827, + 0.03018180839717388, + 0.05713042616844177, + -3.0162190963798245, + -0.10986652225255966, + -0.5071755647659302, + 1.0 + ], + [ + 0.3230076730251312, + 0.024358274415135384, + 0.05917001888155937, + -3.020248631136962, + -0.1377440094947815, + -0.4765060245990753, + 1.0 + ], + [ + 0.32416650652885437, + 0.025368360802531242, + 0.05528208240866661, + -3.016849437850066, + -0.12668383121490479, + -0.45763632655143743, + 1.0 + ], + [ + 0.3385603427886963, + 0.014201849699020386, + 0.05941886454820633, + -3.008824559050151, + -0.13620562851428986, + -0.4727837145328521, + 1.0 + ], + [ + 0.3528946340084076, + 0.00016132387099787593, + 0.06150909140706062, + -3.0186703522973737, + -0.14771324396133426, + -0.4779397249221802, + 1.0 + ], + [ + 0.36202624440193176, + -0.009337162598967552, + 0.05902022495865822, + -3.0080423672967633, + -0.13624709844589233, + -0.49041393399238586, + 1.0 + ], + [ + 0.36546921730041504, + -0.019413908943533897, + 0.05659165978431702, + -3.006543370085307, + -0.13140754401683807, + -0.5103792548179626, + 1.0 + ], + [ + 0.3708011209964752, + -0.026063716039061546, + 0.05908142775297165, + -3.0175134931975087, + -0.14649216830730438, + -0.5086431503295898, + 1.0 + ], + [ + 0.3698829412460327, + -0.029175668954849243, + 0.05912591144442558, + -3.0418902580910405, + -0.1522093117237091, + -0.4832437038421631, + 1.0 + ], + [ + 0.3628089129924774, + -0.025285327807068825, + 0.06343386322259903, + -3.037853369610854, + -0.13463276624679565, + -0.42159748077392584, + 1.0 + ], + [ + 0.3518187999725342, + -0.017339108511805534, + 0.08206818252801895, + -3.0399724935465535, + -0.13350263237953186, + -0.3561947941780091, + 1.0 + ], + [ + 0.34646978974342346, + 0.0008248609956353903, + 0.08293944597244263, + -3.050536984699317, + -0.11349813640117645, + -0.3086206614971161, + 1.0 + ], + [ + 0.3398802578449249, + 0.017064785584807396, + 0.08321848511695862, + -3.0888917316966733, + -0.07883016765117645, + -0.21069921553134915, + 1.0 + ], + [ + 0.3394192159175873, + 0.04663635417819023, + 0.08947508782148361, + 3.1277903785281858, + -0.09008311480283739, + -0.03081355057656765, + 1.0 + ], + [ + 0.34319719672203064, + 0.05871884524822235, + 0.09972329437732697, + 3.1012823676043233, + -0.10141646862030029, + 0.0712277889251709, + 1.0 + ], + [ + 0.3419226109981537, + 0.08625572919845581, + 0.10135282576084137, + 3.1128831171714744, + -0.1072869747877121, + 0.18343690037727353, + 1.0 + ], + [ + 0.3401920795440674, + 0.10137296468019485, + 0.09866224974393845, + 3.1069306788318833, + -0.10599936544895172, + 0.27048081159591675, + 1.0 + ], + [ + 0.3397789001464844, + 0.11439134180545807, + 0.10375117510557175, + 3.081679446744271, + -0.11791146546602249, + 0.32210147380828863, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516, + 0.0005114078521728516 + ], + "episode_id": "1996", + "latent_videos": [ + { + "latent_video_path": "latent_videos/val/1996/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/2390.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/2390.json new file mode 100644 index 00000000..51242622 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/2390.json @@ -0,0 +1,622 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "Take the pot and move it below the microwave" + ], + "videos": [ + { + "video_path": "videos/val/2390/rgb.mp4" + } + ], + "action": [ + [ + -0.0002169229755563324, + 0.00011002957926784335, + -1.855050449142695e-07, + -0.000691772211654791, + -0.0013690793964152043, + -0.0015335049554272492, + 1.0 + ], + [ + -0.0004053284773168887, + 0.0033517733099571465, + 0.00681062009500477, + -0.015436572358413376, + -0.019193860319629722, + 0.0077891510127814576, + 1.0 + ], + [ + 0.01538310628580405, + 0.03173720961139567, + -0.0021340172191341216, + -0.02069496013962489, + 0.009761906968794502, + 0.12373004272602603, + 1.0 + ], + [ + 0.012967805286189201, + 0.01762769436328063, + -0.0031040833745352185, + -0.018202705336447772, + 0.019810046947099952, + 0.0631046269667766, + 1.0 + ], + [ + 0.035082675754124486, + 0.04382662636826047, + -0.01768629101949191, + -0.05021076784787732, + 0.1013309786721476, + 0.1822358430917308, + 1.0 + ], + [ + 0.028827079409011226, + 0.03661044323259952, + -0.010089472885745056, + -0.03248727724543948, + 0.07173096863693572, + 0.16061691445844414, + 1.0 + ], + [ + 0.03497908815531214, + 0.03715336703580041, + 0.00011960277887869961, + 0.003669867352172978, + 0.035298907255478, + 0.13578736571775843, + 1.0 + ], + [ + 0.007664419922574201, + 0.002722061936638623, + 0.0324599878652268, + 0.002859123496340325, + -0.0767243479297258, + 0.008280491631045826, + 1.0 + ], + [ + -0.0031009603915317916, + -0.0023315355177998824, + 0.032088220320093036, + 0.04088897428162119, + -0.09603539262499769, + -0.025549191201217902, + 1.0 + ], + [ + -0.005242535822931501, + -0.010121134777147586, + 0.014246356277278938, + 0.06341402654977249, + -0.05323484116130707, + -0.07388704576869412, + 1.0 + ], + [ + -0.0011952342688305287, + 0.00017487196366890368, + 0.0005238291337368302, + 0.007783378112512732, + -0.0038016727087638864, + -0.002095436522931706, + 1.0 + ], + [ + -0.0015424136627896646, + 0.006578093597899579, + 0.0009278950183115824, + -0.007209302876653185, + 0.013211967349079298, + 0.0022895552674084257, + 1.0 + ], + [ + -0.0022678701476633477, + 0.0035104877922201316, + -0.000643140885010247, + -0.01221230937730758, + 0.010537357170680469, + 0.0035259212841178425, + 0.0 + ], + [ + -0.0011874578754595987, + 0.004480214459246992, + -0.0050515710436301615, + -0.01876815653617142, + 0.02114207012907284, + -0.0024061550262040004, + 0.0 + ], + [ + -0.004208305220262678, + 0.0012170120060052414, + -0.020203514041491212, + -0.005493510216797091, + 0.042166495822007655, + -0.024948259590953835, + 0.0 + ], + [ + -0.005735163656145989, + -0.009250999875367143, + -0.03726761961866402, + 0.028905324809944552, + 0.08804969685262228, + -0.08122478485635659, + 0.0 + ], + [ + -0.015428628417766243, + -0.030633214093291845, + -0.04150612283021712, + 0.056060691720713075, + 0.07673384379498034, + -0.1384469249123161, + 0.0 + ], + [ + -0.017735537997965872, + -0.04620785294500512, + -0.0037135503451808283, + 0.06268855203785635, + -0.011952383089127617, + -0.14201849727417773, + 0.0 + ], + [ + 0.00020748737902496174, + -0.04524312311908337, + 0.029278471903167992, + 0.07674266652623626, + -0.07738391675149353, + -0.07773888119277507, + 0.0 + ], + [ + 0.0088994063003886, + -0.03356763949052341, + 0.04030966681487722, + 0.01792072681710663, + -0.07206644413982305, + -0.038580946020363584, + 0.0 + ], + [ + 0.013836655990433289, + -0.010341967970547093, + 0.020705492107549284, + 0.013918672121694377, + -0.04107619428144975, + -0.00028303509953476596, + 0.0 + ], + [ + 0.00558730024554779, + -0.005860908180191905, + 0.011425128624836483, + 0.015698687669584207, + -0.03267188181811408, + 0.012328471802178956, + 0.0 + ], + [ + -0.0020415551029056935, + -0.004155584261254021, + 0.012353571485738724, + -0.0018649832378395893, + -0.0349760223940667, + -0.004240098100716135, + 0.0 + ], + [ + -0.002206040093466402, + 0.000599615251111404, + 0.010137744550969505, + -0.006056804426959922, + -0.044086125671263826, + 0.00322476972138846, + 0.0 + ], + [ + -0.0019791603820045285, + 0.0003808704863376483, + -0.0006209678949885426, + -0.004292362342673826, + -0.014852490050244454, + 0.006182293205724707, + 1.0 + ], + [ + -0.008485946768424688, + 0.0043824783803503445, + -0.00909057203932112, + -0.01153438927347383, + 0.04012535873534723, + -0.012910794338471141, + 1.0 + ], + [ + -0.015214348951946538, + -0.008719520837136932, + -0.035140528125863595, + -0.0164312389043502, + 0.10961714579366923, + -0.10093407702228877, + 1.0 + ], + [ + -0.01152557457544042, + -0.0005852767555287303, + -0.01702933183374521, + -0.04825522255448574, + 0.04009572318131238, + -0.06676902980826512, + 1.0 + ], + [ + -0.0046653846903735195, + 0.0038829476775773695, + -0.0016465532197798467, + -0.03636430108166705, + -0.005197068774601317, + -0.011004919668619634, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 2.3519839316112146e-09, + 6.5267588000044506e-09, + 0.0015339821790231847, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.1443345307382643e-17, + 2.784606887626297e-17, + -1.064055663618102e-17, + 1.0 + ] + ], + "state": [ + [ + 0.2714306712150574, + 0.09608350694179535, + 0.04992594197392464, + -3.0587216412001332, + -0.028593027964234352, + -0.2082616686820984, + 1.0 + ], + [ + 0.2711960971355438, + 0.09602098912000656, + 0.04991082102060318, + -3.0594603960686406, + -0.027355546131730076, + -0.20661948621273038, + 1.0 + ], + [ + 0.2704179286956787, + 0.09334191679954529, + 0.04283973574638367, + -3.0747141187363347, + -0.007586853578686714, + -0.21280652284622192, + 1.0 + ], + [ + 0.27873536944389343, + 0.05900172144174576, + 0.04296473413705826, + -3.094981862353631, + -0.00902118906378746, + -0.33691510558128357, + 1.0 + ], + [ + 0.28508540987968445, + 0.03796675428748131, + 0.0453609935939312, + -3.1126940363370856, + -0.02585301175713539, + -0.4008849561214447, + 1.0 + ], + [ + 0.29969027638435364, + -0.016359085217118263, + 0.06267495453357697, + 3.1247251016371926, + -0.1214638203382492, + -0.5864543914794922, + 1.0 + ], + [ + 0.3022797107696533, + -0.06182398274540901, + 0.07679446041584015, + 3.1120107137882194, + -0.19432090222835544, + -0.7484830617904663, + 1.0 + ], + [ + 0.30201345682144165, + -0.11226528882980347, + 0.08450985699892044, + -3.1406083602975166, + -0.2318125218153, + -0.8867945671081543, + 1.0 + ], + [ + 0.3093548119068146, + -0.12552693486213684, + 0.054676368832588196, + -3.1358385003455957, + -0.1550722569227219, + -0.8950746059417723, + 1.0 + ], + [ + 0.3125002682209015, + -0.1254280060529709, + 0.022508027032017708, + -3.0989630987220487, + -0.05913298949599266, + -0.869046151638031, + 1.0 + ], + [ + 0.3178347051143646, + -0.11513440310955048, + 0.008420240134000778, + -3.0401044731312474, + -0.008921478874981405, + -0.7930541038513184, + 1.0 + ], + [ + 0.31691330671310425, + -0.11437109112739563, + 0.007870747707784176, + -3.032342704134532, + -0.005351637490093708, + -0.7905842661857606, + 1.0 + ], + [ + 0.31125882267951965, + -0.11780952662229538, + 0.006222926080226898, + -3.039523298042365, + -0.018235160037875175, + -0.7943010926246643, + 1.0 + ], + [ + 0.3071281909942627, + -0.11868149787187576, + 0.006463628727942705, + -3.051647426682063, + -0.028358219191432, + -0.7988839745521544, + 0.0 + ], + [ + 0.3026861250400543, + -0.12116573005914688, + 0.011056818068027496, + -3.070409545796462, + -0.049630939960479736, + -0.7983863949775697, + 0.0 + ], + [ + 0.2971595823764801, + -0.11929227411746979, + 0.030889125540852547, + -3.076947109895297, + -0.09345293790102004, + -0.7764265537261963, + 0.0 + ], + [ + 0.29535210132598877, + -0.10795153677463531, + 0.0679764449596405, + -3.0551248659663877, + -0.186292365193367, + -0.7000606656074523, + 0.0 + ], + [ + 0.29487690329551697, + -0.0723339393734932, + 0.1083541065454483, + -3.0247518589072904, + -0.2730117440223694, + -0.5641602277755737, + 0.0 + ], + [ + 0.3026847541332245, + -0.023478159680962563, + 0.11231083422899246, + -3.0030238052182874, + -0.27480185031890875, + -0.4161262214183808, + 0.0 + ], + [ + 0.32826119661331177, + 0.018630361184477806, + 0.09047210216522217, + -2.9506829251819333, + -0.20782046020030975, + -0.3267219364643097, + 0.0 + ], + [ + 0.3560299873352051, + 0.05209517851471901, + 0.05981504172086716, + -2.9432622512155255, + -0.1441054791212082, + -0.274724543094635, + 0.0 + ], + [ + 0.3755912482738495, + 0.06135468930006027, + 0.04372948035597801, + -2.9303902109437665, + -0.10388565063476562, + -0.26631075143814087, + 0.0 + ], + [ + 0.3840857148170471, + 0.06746018677949905, + 0.03441987931728363, + -2.9140391071611127, + -0.06935412436723709, + -0.271523118019104, + 0.0 + ], + [ + 0.3846977949142456, + 0.07438509166240692, + 0.023207448422908783, + -2.916612895327159, + -0.03623158112168313, + -0.25949788093566895, + 0.0 + ], + [ + 0.3833480179309845, + 0.07647862285375595, + 0.013118085451424122, + -2.9226948787742337, + 0.007463010028004646, + -0.2528062462806702, + 0.0 + ], + [ + 0.3813088834285736, + 0.0764821395277977, + 0.013656283728778362, + -2.9269845505529126, + 0.023303532972931862, + -0.25561591982841486, + 1.0 + ], + [ + 0.371707946062088, + 0.07256444543600082, + 0.021800581365823746, + -2.938250424461909, + -0.01865057833492756, + -0.25155407190322876, + 1.0 + ], + [ + 0.35668084025382996, + 0.07791674882173538, + 0.05768793448805809, + -2.9554909636550626, + -0.1463126689195633, + -0.17465521395206454, + 1.0 + ], + [ + 0.3425860106945038, + 0.07778827846050262, + 0.07267121970653534, + -3.01256416936452, + -0.19781222939491272, + -0.11536718904972075, + 1.0 + ], + [ + 0.3373531401157379, + 0.07430495321750641, + 0.07286550104618073, + -3.0512550865584096, + -0.19406090676784518, + -0.1035635769367218, + 1.0 + ], + [ + 0.3373531401157379, + 0.07430495321750641, + 0.07286550104618073, + -3.050954947369643, + -0.19392229616641998, + -0.10512048751115799, + 1.0 + ], + [ + 0.3373531401157379, + 0.07430495321750641, + 0.07286550104618073, + -3.050954947369643, + -0.19392229616641998, + -0.10512048751115799, + 1.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.05002796649932861, + 0.3288310766220093, + 0.6321892142295837, + 0.8412791043519974, + 0.9207848757505417, + 0.9257014021277428, + 0.9257014021277428, + 0.9257014021277428, + 0.9257014021277428, + 0.9257014021277428, + 0.9257014021277428, + 0.9257014021277428, + 0.8904407843947411, + 0.7226157188415527, + 0.437961220741272, + 0.13836663961410522, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "episode_id": "2390", + "latent_videos": [ + { + "latent_video_path": "latent_videos/val/2390/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/27.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/27.json new file mode 100644 index 00000000..ae22e83c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/annotation/val/27.json @@ -0,0 +1,470 @@ +{ + "task": "robot_trajectory_prediction", + "texts": [ + "pick up pot 50" + ], + "videos": [ + { + "video_path": "videos/val/27/rgb.mp4" + } + ], + "action": [ + [ + -0.00024484915857862094, + 0.0004511619871172971, + 0.000384019201372169, + 5.411372109142075e-05, + -0.0015363687295025337, + 0.0015305880057052456, + 1.0 + ], + [ + 0.0013726356021386938, + 0.002854295264867983, + 0.013462637969405403, + -0.034436124626627714, + -0.05595909346119582, + -0.030530909129315824, + 1.0 + ], + [ + 0.0017037976352665676, + -0.002227465596975925, + 0.011947803462860283, + -0.0439517071307138, + -0.04660450421762375, + -0.045530014454148345, + 1.0 + ], + [ + -0.0005689494235845633, + 0.005574537794757489, + 0.0024681516126573017, + -0.008499936803744237, + -0.012068386111205856, + 0.020782726145737652, + 1.0 + ], + [ + 0.002186039524930557, + -0.0032468973681446934, + 0.0029279936077351914, + 0.03136513224268517, + 0.006736078166670399, + -0.0008528058175395837, + 1.0 + ], + [ + -0.002492361723184521, + 0.0037768484568409134, + 0.014739484440779289, + 0.018708511393383524, + 0.04669310178739547, + 0.12036238348148824, + 1.0 + ], + [ + -0.0010868493355973322, + -0.00037318912410315403, + 0.003220235007407957, + -0.059839726540418955, + -0.06498783397793217, + 0.074342966384538, + 1.0 + ], + [ + 0.00531809083081411, + -0.006180470679063724, + 0.01757052382179307, + -0.05734331362975081, + -0.16535275455399484, + -0.03551142738725975, + 1.0 + ], + [ + 0.005398037093388957, + -0.0061498812675467775, + 0.00961448237844452, + -0.010402725280951812, + 0.030390977782139672, + -0.12365481455556637, + 1.0 + ], + [ + -0.0018167523141190286, + -0.012579415927181283, + 0.01733363031049781, + 0.019409409833032184, + 0.018615339001111824, + -0.09637254282354189, + 1.0 + ], + [ + -0.0063659336239673256, + -0.0004424777813890691, + 0.0014571810279801287, + 0.06033229616852111, + 0.07523749083503027, + -0.04854317181987876, + 1.0 + ], + [ + -0.00503010731663823, + -0.0014644515742708508, + 0.0004219593731718887, + 0.02522916144174192, + 0.094213002979486, + -0.05361771081489881, + 1.0 + ], + [ + -0.002663418549376775, + 0.0003227599902892499, + 0.00817350179145052, + -0.0059319910427376235, + 0.026961810502427096, + -0.04623098949062257, + 1.0 + ], + [ + 0.003096223052293713, + 0.0044564460297623135, + 0.005038557496201554, + -0.023739531135132715, + -0.00016069651916667506, + -0.039423609826271216, + 1.0 + ], + [ + 0.004257900628960489, + 0.00665191989449729, + 0.012944410365677107, + -0.026641659186868505, + -0.031096627049864555, + -0.05550613746491933, + 1.0 + ], + [ + 0.011008017025251542, + 0.0101149568257008, + 0.0036519839245758635, + -0.00885252803725942, + 0.01698003938363803, + 0.009453528990087338, + 1.0 + ], + [ + 0.006369282997067744, + 0.007644821505806194, + -0.0006699734742781573, + -0.022797662920623775, + 0.03913675196542728, + 0.047740738618923244, + 0.0 + ], + [ + -0.00015362606229642341, + 0.0019354077356836601, + -0.0005156574998426497, + -0.013901212661313908, + 0.01701455423503176, + 0.025645210227297297, + 0.0 + ], + [ + 0.001047703328587518, + 0.0020913995794681227, + -0.0017642325467847934, + -0.004972977399192736, + 0.014143697720507728, + -0.00022905215190250147, + 0.0 + ], + [ + -0.001285485220675486, + 0.0009263511287794915, + 3.179031324279672e-05, + 0.004988568085955926, + -0.002486797649164113, + 0.011298914693506459, + 0.0 + ], + [ + 9.996373918494515e-05, + 0.0018644814889715565, + -0.005635180628709158, + -0.01293900040052292, + 0.0157859798261856, + 0.007998549949977795, + 0.0 + ], + [ + 0.0013674128518444688, + -8.135908266406197e-05, + -0.0065647821947801485, + -0.0017956860926352568, + 0.009181824918700501, + 0.01366550381024103, + 0.0 + ], + [ + 0.01368349963347882, + 0.004776720171973794, + -0.03950541337555147, + -0.014629549637008986, + 0.1123585445379473, + 0.022574478104602495, + 0.0 + ] + ], + "state": [ + [ + 0.29474949836730957, + -0.0015996829606592655, + 0.19281825423240662, + 3.14005875762599, + 0.035281602293252945, + -0.004599587991833687, + 1.0 + ], + [ + 0.29448920488357544, + -0.0020502409897744656, + 0.19244380295276642, + 3.140058761234865, + 0.03681557998061181, + -0.006133568473160267, + 1.0 + ], + [ + 0.2953478991985321, + -0.004930504132062197, + 0.1789441555738449, + 3.106746209906884, + 0.09280432760715486, + 0.024395082145929337, + 1.0 + ], + [ + 0.2958863079547882, + -0.0031069691758602858, + 0.16681982576847076, + 3.0668959115915975, + 0.14087627828121185, + 0.0686621144413948, + 1.0 + ], + [ + 0.29543185234069824, + -0.008894971571862698, + 0.16487471759319305, + 3.0553367381268224, + 0.1513269692659378, + 0.046788029372692115, + 1.0 + ], + [ + 0.2969696521759033, + -0.005837118253111839, + 0.16138488054275513, + 3.0869178382032594, + 0.14468924701213837, + 0.04823304712772369, + 1.0 + ], + [ + 0.29263702034950256, + -0.01062826719135046, + 0.14738480746746063, + 3.0889509861641606, + 0.09050387889146805, + -0.06973318755626678, + 1.0 + ], + [ + 0.29127970337867737, + -0.010329736396670341, + 0.14426086843013763, + 3.022066543000289, + 0.1512189358472824, + -0.14813946187496185, + 1.0 + ], + [ + 0.2943669855594635, + -0.006704472471028566, + 0.12548509240150452, + 2.9655139615112027, + 0.3195969462394715, + -0.13218846917152405, + 1.0 + ], + [ + 0.29673945903778076, + -0.002610838506370783, + 0.11377989500761032, + 2.998258026438304, + 0.30861714482307434, + 0.0011890217429026963, + 1.0 + ], + [ + 0.28923994302749634, + 0.00735465670004487, + 0.09627437591552734, + 3.049484061198779, + 0.3024081587791443, + 0.10390132665634157, + 1.0 + ], + [ + 0.28272199630737305, + 0.0069832210429012775, + 0.09674626588821411, + 3.126596617633947, + 0.23148658871650699, + 0.16053688526153567, + 1.0 + ], + [ + 0.2775554358959198, + 0.007623548619449139, + 0.09746826440095901, + -3.1186644678288182, + 0.13773876428604123, + 0.2158477008342743, + 1.0 + ], + [ + 0.2739105224609375, + 0.0066858502104878426, + 0.08973266929388046, + -3.118315115692564, + 0.10958096385002136, + 0.2617062628269196, + 1.0 + ], + [ + 0.2774626612663269, + 0.003146409522742033, + 0.0842839777469635, + -3.137737549962946, + 0.10873880237340926, + 0.301357239484787, + 1.0 + ], + [ + 0.28212010860443115, + -0.0023194742389023304, + 0.07092853635549545, + 3.1248948016488036, + 0.13945190608501434, + 0.357501745223999, + 1.0 + ], + [ + 0.2954389750957489, + -0.008205404505133629, + 0.06594964116811752, + 3.114756543310829, + 0.12231050431728362, + 0.34826540946960444, + 1.0 + ], + [ + 0.30408334732055664, + -0.013177972286939621, + 0.06604088097810745, + 3.0862560143047055, + 0.081774078309536, + 0.3014730215072632, + 0.0 + ], + [ + 0.30455097556114197, + -0.015026410110294819, + 0.06667326390743256, + 3.070345068471976, + 0.063342422246933, + 0.2767625153064728, + 0.0 + ], + [ + 0.30620869994163513, + -0.01659359037876129, + 0.06851175427436829, + 3.0654433016949376, + 0.04925086721777916, + 0.27799928188323975, + 0.0 + ], + [ + 0.305230051279068, + -0.0178359467536211, + 0.06861376762390137, + 3.069871964054652, + 0.05086766928434372, + 0.2665291130542755, + 0.0 + ], + [ + 0.305992066860199, + -0.019137060269713402, + 0.07435554265975952, + 3.0565773641043386, + 0.03454791754484177, + 0.2596792280673981, + 0.0 + ], + [ + 0.3073670268058777, + -0.01811111345887184, + 0.08083857595920563, + 3.0543416013293943, + 0.024236060678958893, + 0.24683959782123566, + 0.0 + ], + [ + 0.3218880295753479, + -0.015809306874871254, + 0.12026667594909668, + 3.038856068747588, + -0.08966232091188431, + 0.23421402275562286, + 0.0 + ] + ], + "continuous_gripper_state": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.18319201469421387, + 0.4596704840660095, + 0.6988680958747864, + 0.886321172118187, + 0.9349767416715622, + 0.9364518448710442, + 0.9373273998498917, + 0.9373273998498917 + ], + "episode_id": "27", + "latent_videos": [ + { + "latent_video_path": "latent_videos/val/27/0.pt" + } + ] +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/2285/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/2285/rgb.mp4 new file mode 100644 index 00000000..14cb6a83 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/2285/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/2285/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/2285/state.npy new file mode 100644 index 00000000..cf47deea Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/2285/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/312/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/312/rgb.mp4 new file mode 100644 index 00000000..8da89f6c Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/312/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/312/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/312/state.npy new file mode 100644 index 00000000..88ad20b6 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/312/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/3266/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/3266/rgb.mp4 new file mode 100644 index 00000000..4a702c96 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/3266/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/3266/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/3266/state.npy new file mode 100644 index 00000000..231d4001 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/3266/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/43/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/43/rgb.mp4 new file mode 100644 index 00000000..1d974c86 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/43/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/43/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/43/state.npy new file mode 100644 index 00000000..0ffce64e Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/43/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/570/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/570/rgb.mp4 new file mode 100644 index 00000000..bd0a2662 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/570/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/570/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/570/state.npy new file mode 100644 index 00000000..6231311c Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/test/570/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/13109/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/13109/rgb.mp4 new file mode 100644 index 00000000..afd4283d Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/13109/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/13109/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/13109/state.npy new file mode 100644 index 00000000..4d8d1ef6 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/13109/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/1432/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/1432/rgb.mp4 new file mode 100644 index 00000000..06af2367 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/1432/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/1432/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/1432/state.npy new file mode 100644 index 00000000..d1b8cbe2 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/1432/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/19139/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/19139/rgb.mp4 new file mode 100644 index 00000000..7391aa51 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/19139/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/19139/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/19139/state.npy new file mode 100644 index 00000000..943767d0 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/19139/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/2277/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/2277/rgb.mp4 new file mode 100644 index 00000000..1507cc33 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/2277/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/2277/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/2277/state.npy new file mode 100644 index 00000000..ad33fcff Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/2277/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/23977/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/23977/rgb.mp4 new file mode 100644 index 00000000..bed0f513 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/23977/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/23977/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/23977/state.npy new file mode 100644 index 00000000..df6c2b47 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/23977/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/346/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/346/rgb.mp4 new file mode 100644 index 00000000..7e5f83d8 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/346/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/346/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/346/state.npy new file mode 100644 index 00000000..5b2d1737 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/346/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/4682/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/4682/rgb.mp4 new file mode 100644 index 00000000..bd7eb81e Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/4682/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/4682/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/4682/state.npy new file mode 100644 index 00000000..5bd72290 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/4682/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/5873/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/5873/rgb.mp4 new file mode 100644 index 00000000..6881c4e8 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/5873/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/5873/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/5873/state.npy new file mode 100644 index 00000000..69ef905e Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/5873/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/83/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/83/rgb.mp4 new file mode 100644 index 00000000..23667467 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/83/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/83/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/83/state.npy new file mode 100644 index 00000000..d1bdac45 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/83/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/9496/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/9496/rgb.mp4 new file mode 100644 index 00000000..307a0d44 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/9496/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/9496/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/9496/state.npy new file mode 100644 index 00000000..95864105 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/train/9496/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1527/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1527/rgb.mp4 new file mode 100644 index 00000000..2f7174fc Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1527/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1527/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1527/state.npy new file mode 100644 index 00000000..de6fd377 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1527/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/199/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/199/rgb.mp4 new file mode 100644 index 00000000..0f33d1c4 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/199/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/199/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/199/state.npy new file mode 100644 index 00000000..1a4c9e14 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/199/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1996/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1996/rgb.mp4 new file mode 100644 index 00000000..9779124c Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1996/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1996/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1996/state.npy new file mode 100644 index 00000000..ea67d939 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/1996/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/2390/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/2390/rgb.mp4 new file mode 100644 index 00000000..0caee088 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/2390/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/2390/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/2390/state.npy new file mode 100644 index 00000000..9257dacc Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/2390/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/27/rgb.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/27/rgb.mp4 new file mode 100644 index 00000000..649b0f25 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/27/rgb.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/27/state.npy b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/27/state.npy new file mode 100644 index 00000000..d4b9079f Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/bridge/videos/val/27/state.npy differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/nemo/finetuned_result.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/nemo/finetuned_result.mp4 new file mode 100644 index 00000000..7a50b839 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/nemo/finetuned_result.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/0.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/0.mp4 new file mode 100644 index 00000000..210c51cc Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/0.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/1.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/1.mp4 new file mode 100644 index 00000000..818164da Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/1.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/2.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/2.mp4 new file mode 100644 index 00000000..5bcb73f7 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/2.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/3.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/3.mp4 new file mode 100644 index 00000000..731783dc Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/3.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/4.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/4.mp4 new file mode 100644 index 00000000..5cbfce86 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/4.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/5.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/5.mp4 new file mode 100644 index 00000000..e9e3fb9c Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/5.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/6.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/6.mp4 new file mode 100644 index 00000000..e1620d02 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/6.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/7.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/7.mp4 new file mode 100644 index 00000000..f493a11b Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/7.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/8.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/8.mp4 new file mode 100644 index 00000000..37b29461 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/8.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/9.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/9.mp4 new file mode 100644 index 00000000..6317d1d5 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/9.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/base.jsonl b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/base.jsonl new file mode 100644 index 00000000..3ef2e68b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/base.jsonl @@ -0,0 +1,10 @@ +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/0.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/1.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/2.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/3.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/4.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/5.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/6.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/7.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/8.mp4"} +{"visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/9.mp4"} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/video2world.jsonl b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/video2world.jsonl new file mode 100644 index 00000000..6bb3a542 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/batch_inputs/video2world.jsonl @@ -0,0 +1,10 @@ +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/0.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/1.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/2.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/3.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/4.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/5.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/6.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/7.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/8.mp4"} +{"prompt": "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions.", "visual_input": "cosmos1/models/autoregressive/assets/v1p0/batch_inputs/9.mp4"} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/input.jpg b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/input.jpg new file mode 100644 index 00000000..1c166b2c Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/input.jpg differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/input.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/input.mp4 new file mode 100644 index 00000000..e9e3fb9c Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/input.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_image_input_12b.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_image_input_12b.mp4 new file mode 100644 index 00000000..46f45939 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_image_input_12b.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_image_input_13b.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_image_input_13b.mp4 new file mode 100644 index 00000000..fb35337a Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_image_input_13b.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_video_input_12b.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_video_input_12b.mp4 new file mode 100644 index 00000000..60b1270b Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_video_input_12b.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_video_input_13b.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_video_input_13b.mp4 new file mode 100644 index 00000000..9a2c745e Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/assets/v1p0/output_from_video_input_13b.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/model.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/model.py new file mode 100644 index 00000000..584e24ca --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/model.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0301 + +from typing import Optional + +import attrs + +from cosmos1.models.autoregressive.configs.base.tokenizer import TokenizerConfig + + +@attrs.define +class ModelConfig: + """ + A class to hold model configuration arguments. + + Args: + dim (int): The dimensionality of the input and output of each transformer block. + n_layers (int): Number of layers in the transformer. + n_heads (int): Number of attention heads. + n_kv_heads (Optional[int]): Number of key-value heads. If None, defaults to n_heads. Note: this is equivalent to + `num_gqa_groups` in TransformerEngine, where GQA means Grouped Query Attention. + head_dim (Optional[int]): Dimensionality of each head. If None, defaults to dim // n_heads. + vocab_size (int): Vocabulary size. + ffn_hidden_size (int): Hidden size for feedforward network. + norm_eps (float): Epsilon value for normalization. + rope_theta (float): Theta value for rotary positional embeddings. + apply_abs_pos_emb (bool): Whether to apply absolute position embeddings. + max_batch_size (int): Maximum batch size for inference. + max_seq_len (int): Maximum sequence length for input text. + fuse_qkv (bool): Whether to fuse QKV in attention. Defaults to True. + causal_mask (bool): Whether to use causal mask. Defaults to True. + norm_type (str): Type of normalization layer. Choices: "rmsnorm", "fused_rmsnorm", "layernorm", "np_layernorm". + precision (str): Data type for the model. + use_qk_normalization (bool): Whether to enable QK normalization. + ckpt_dir (str): Checkpoint directory. + ckpt_path (str): Checkpoint path. + apply_yarn (Optional[bool]): Whether to apply YaRN (long-context extension). + yarn_scale (Optional[float]): Scale factor for YaRN. + yarn_beta_fast (Optional[int]): Beta fast variable for YaRN (i.e., low_freq_factor in Llama 3.1 RoPE scaling code) + yarn_beta_slow (Optional[int]): Beta slow variable for YaRN (i.e., high_freq_factor in Llama 3.1 RoPE scaling code) + original_seq_len (Optional[int]): Original sequence length. + vision_encoder (Optional[str]): Vision encoder name. + mm_projector (Optional[str]): Multi-modal projector name. + vision_encoder_in_channels (Optional[int]): Number of channels in the input image for the vision encoder. Default is 3, you can specify to int larger than 3. E.g. if you have 4-channel images with the last channel as the alpha channel, set this to 4. + rope_dim (Optional[str]): Dimensionality of the RoPE. Choices: "1D", "3D". + pytorch_rope_version (Optional[str]): Version of the PyTorch RoPE implementation. Choices: "v1", "v2". + original_latent_shape (Optional[list]): Original shape of the latent tensor needed for rope extension. + pad_to_multiple_of (Optional[int]): Pad the position embedding to a multiple of this value. + vision_encoder_in_channels (Optional[int]): Number of channels in the input image for the vision encoder. Default is 3. + insert_cross_attn (bool): Whether to insert the cross-attention layers after each multi-head self-attention (MSA) layer. + insert_cross_attn_every_k_layers (int): Insert cross-attention layers every k TransformerLayers. + context_dim (Optional[int]): The dimensionality of cross-attention embedding, e.g., T5 embed feature dim. + num_video_frames (Optional[int]): Number of video frames. + video_height (Optional[int]): Raw video pixel height dimension. + video_width (Optional[int]): Raw video pixel width dimension. + video_latent_shape (Optional[list]): Video tokenizer output dimension, in (T,H,W). + """ + + dim: int = attrs.field(default=4096) + n_layers: int = attrs.field(default=32) + n_heads: int = attrs.field(default=32) + n_kv_heads: Optional[int] = attrs.field(default=8) + head_dim: Optional[int] = attrs.field(default=None) + vocab_size: int = attrs.field(default=128256) + ffn_hidden_size: int = attrs.field(default=14336) + norm_eps: float = attrs.field(default=1e-5) + rope_theta: float = attrs.field(default=500000) + apply_abs_pos_emb: bool = attrs.field(default=False) + max_batch_size: int = attrs.field(default=1) + max_seq_len: int = attrs.field(default=8192) + fuse_qkv: bool = attrs.field(default=False) + causal_mask: bool = attrs.field(default=True) + norm_type: str = attrs.field(default="rmsnorm") + precision: str = attrs.field(default="bfloat16") + use_qk_normalization: bool = False + tokenizer: Optional[TokenizerConfig] = None + ckpt_dir: Optional[str] = attrs.field(default=None) + ckpt_path: Optional[str] = attrs.field( + default=None + ) # If not None, load the model from this path instead of ckpt_dir + apply_yarn: Optional[bool] = attrs.field(default=False) + yarn_scale: Optional[float] = attrs.field(default=None) + yarn_beta_fast: Optional[int] = attrs.field(default=None) + yarn_beta_slow: Optional[int] = attrs.field(default=None) + original_seq_len: Optional[int] = attrs.field(default=None) + vision_encoder: Optional[str] = attrs.field(default=None) + vision_encoder_in_channels: Optional[int] = attrs.field(default=3) + mm_projector: Optional[str] = attrs.field(default=None) + rope_dim: Optional[str] = attrs.field(default="1D") + pytorch_rope_version: Optional[str] = attrs.field(default="v2") + original_latent_shape: Optional[list] = None + pad_to_multiple_of: Optional[int] = None + vision_encoder_in_channels: Optional[int] = attrs.field(default=3) + insert_cross_attn: bool = False + insert_cross_attn_every_k_layers: int = 1 + context_dim: Optional[int] = attrs.field(default=1024) + # For video training + num_video_frames: Optional[int] = None + # Raw video pixel dimension + video_height: Optional[int] = None + video_width: Optional[int] = None + # Video tokenizer output dimension, in (T,H,W), it's computed by num_video_frames/temporal_compress_factor, video_height/spatial_compression_fact, video_width/spatial_compression_fact + video_latent_shape: Optional[list] = None + + def __getitem__(self, item): + return getattr(self, item) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/model_config.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/model_config.py new file mode 100644 index 00000000..306f08f8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/model_config.py @@ -0,0 +1,424 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0301 + +import copy +from typing import Callable, List, Optional + +from cosmos1.models.autoregressive.configs.base.model import ModelConfig +from cosmos1.models.autoregressive.configs.base.tokenizer import ( + TextTokenizerConfig, + TokenizerConfig, + VideoTokenizerConfig, + create_discrete_video_fsq_tokenizer_state_dict_config, +) +from cosmos1.models.autoregressive.tokenizer.image_text_tokenizer import ImageTextTokenizer +from cosmos1.models.autoregressive.tokenizer.text_tokenizer import TextTokenizer +from cosmos1.utils import log +from cosmos1.utils.lazy_config import LazyCall as L + + +# Common architecture specifications +BASE_CONFIG = {"n_kv_heads": 8, "norm_type": "rmsnorm", "norm_eps": 1e-5, "ffn_hidden_size": 14336} +COSMOS_ARCHITECTURES = { + "4b": { + "n_layers": 16, + "dim": 4096, + "n_heads": 32, + }, + "12b": { + "n_layers": 40, + "dim": 5120, + "n_heads": 32, + "head_dim": 128, + }, +} + +COSMOS_YARN_CONFIG = { + "original_latent_shape": [3, 40, 64], + "apply_yarn": True, + "yarn_beta_fast": 4, + "yarn_beta_slow": 1, + "yarn_scale": 2, +} + +# Llama3 architecture specifications for different model sizes +LLAMA3_ARCHITECTURES = { + "8b": { + "n_layers": 32, + "dim": 4096, + "n_heads": 32, + "ffn_hidden_size": 14336, + }, +} +# Llama3.1 uses YaRN for long context support (context of 128k tokens) +LLAMA_YARN_CONFIG = { + "apply_yarn": True, + "yarn_scale": 8, + "yarn_beta_fast": 4, + "yarn_beta_slow": 1, +} + +# Mistral architecture specifications for different model sizes +MISTRAL_ARCHITECTURES = { + "12b": { + "n_layers": 40, + "dim": 5120, + "n_heads": 32, + "ffn_hidden_size": 14336, + "head_dim": 128, + }, +} + +PIXTRAL_VISION_ARCHITECTURES = { + "12b": {"vision_encoder": "pixtral-12b-vit", "mm_projector": "mlp"}, +} + + +def get_model_arch_specs(model_size: str, model_family: str = "mistral", pretrained: bool = False) -> dict: + """ + Get the model architecture specifications for the given model size, model family and pretrained status. + + Args: + model_size (str): Model size. Choices: "1b", "3b", "4b", "7b", etc. + model_family (str): Model family. Choices: "llama", "llama3", "llama3.1", "mistral" + pretrained (bool): Whether to load pretrained weights. + + Returns: + dict: A dictionary containing the model architecture specifications. + """ + arch_specs = copy.deepcopy(BASE_CONFIG) + model_size = model_size.lower() + if model_family.startswith("cosmos"): + arch_specs.update(COSMOS_ARCHITECTURES[model_size]) + elif model_family.startswith("llama"): + arch_specs.update(LLAMA3_ARCHITECTURES[model_size]) + elif model_family in ["mistral", "pixtral"]: + arch_specs.update(MISTRAL_ARCHITECTURES[model_size]) + if model_family == "pixtral": + arch_specs.update(PIXTRAL_VISION_ARCHITECTURES[model_size]) + else: + raise ValueError(f"Model family {model_family} is not supported.") + + if pretrained: + if model_family == "cosmos": + if model_size == "12b": + arch_specs.update(COSMOS_YARN_CONFIG) + log.debug(f"Using YaRN for RoPE extension with config: {COSMOS_YARN_CONFIG}") + else: + pass + elif model_family in ["llama", "llama3"]: + pretrained_specs = { + "rope_theta": 500000, + "max_seq_len": 8192, + "vocab_size": 128256, + } + arch_specs.update(pretrained_specs) + elif model_family == "llama3.1": + pretrained_specs = { + "rope_theta": 500000, + "max_seq_len": 131072, + "original_seq_len": 8192, + "vocab_size": 128256, + **LLAMA_YARN_CONFIG, + } + arch_specs.update(pretrained_specs) + elif model_family == "mistral": + assert model_size == "12b", "We only support Mistral-Nemo-12B model." + pretrained_specs = { + "rope_theta": 1000000, + "max_seq_len": 128000, + "vocab_size": 131072, + } + arch_specs.update(pretrained_specs) + elif model_family == "pixtral": + assert model_size == "12b", "We only support Pixtral 12B model." + pretrained_specs = {"rope_theta": 1000000000, "max_seq_len": 128000, "vocab_size": 131072} + arch_specs.update(pretrained_specs) + else: + raise ValueError(f"Model family {model_family} doesn't have a pretrained config.") + + return arch_specs + + +def create_text_model_config( + model_ckpt_path: str, + tokenizer_path: str, + model_family: str = "mistral", + model_size: str = "12b", + is_instruct_model: bool = True, + max_seq_len: int = None, + max_batch_size: int = 1, + rope_dim: str = "1D", + add_special_tokens: bool = True, + pytorch_rope_version: str = None, +) -> dict: + """Create a text model for training or inference. + Args: + model_ckpt_path (str): Path to the model checkpoint. + tokenizer_path (str): Path to the tokenizer folder. + model_family (str): Model family. Choices: "llama", "llama3", "llama3.1", "mistral". + model_size (str): Model size. Choices: "1b", "3b", "4b", "7b", "8b", "72b", etc. + is_instruct_model (bool): Whether the model is an instruct model. + inference (bool): Whether to create the model for inference. + max_seq_len (int): Maximum sequence length. + max_batch_size (int): Maximum batch size. + rope_dim (str): RoPE dimension. Choices: "1D", "3D". + add_special_tokens (bool): Whether to add special tokens. + Returns: + dict: A dictionary containing the model configuration, which can be used to instantiate the model object. + """ + # Model size specific parameters + model_arch_specs = get_model_arch_specs(model_family=model_family, model_size=model_size, pretrained=True) + if max_seq_len is not None: + # Override the max_seq_len if provided + model_arch_specs["max_seq_len"] = max_seq_len + if pytorch_rope_version is not None: + model_arch_specs["pytorch_rope_version"] = pytorch_rope_version + model_config = ModelConfig( + max_batch_size=max_batch_size, + precision="bfloat16", + ckpt_path=model_ckpt_path, + use_qk_normalization=False, + rope_dim=rope_dim, + **model_arch_specs, + ) + + tokenizer_config = TokenizerConfig( + text_tokenizer=TextTokenizerConfig( + config=L(TextTokenizer)( + model_family=model_family, + is_instruct_model=is_instruct_model, + local_path=tokenizer_path, + ), + data_key="text", + tokenizer_offset=model_config.vocab_size, + tokenize_here=False, + vocab_size=model_config.vocab_size, + ), + seq_len=model_config.max_seq_len, + training_type="text_only", + add_special_tokens=add_special_tokens, + ) + return model_config, tokenizer_config + + +def create_vision_language_model_config( + model_ckpt_path: str, + tokenizer_ckpt_path: str, + model_family: str = "pixtral", + model_size: str = "12b", + is_instruct_model: bool = True, + max_batch_size: int = 1, + rope_dim: str = "1D", + add_special_tokens: bool = True, + max_seq_len: int = None, + vision_encoder_in_channels: int = 3, + fuse_qkv: bool = False, + pytorch_rope_version: str = None, +) -> dict: + """Create a vision-language model for training or inference. + Args: + model_ckpt_path (str): Path to the model checkpoint. + tokenizer_ckpt_path (str): Path to the tokenizer checkpoint. + model_family (str): Model family. Choices: "pixtral". + model_size (str): Model size. Choices: "12b". + is_instruct_model (bool): Whether the model is an instruct model. + rope_dim (str): RoPE dimension. Choices: "1D". + add_special_tokens (bool): Whether to add special tokens. + max_seq_len (int): Maximum sequence length. + vision_encoder_in_channels (int): Number of channels in the input image for the vision encoder. Default is 3, you can specify to int larger than 3. E.g. if you have 4 channel images where last channel is binary mask, set this to 4. + fuse_qkv (bool): Whether to fuse the QKV linear layers. + Returns: + dict: A dictionary containing the model configuration, which can be used to instantiate the model object. + """ + # Model size specific parameters + model_arch_specs = get_model_arch_specs(model_family=model_family, model_size=model_size, pretrained=True) + if max_seq_len is not None: + # Override the max_seq_len if provided + model_arch_specs["max_seq_len"] = max_seq_len + if pytorch_rope_version is not None: + model_arch_specs["pytorch_rope_version"] = pytorch_rope_version + + model_config = ModelConfig( + max_batch_size=max_batch_size, + precision="bfloat16", + ckpt_path=model_ckpt_path, + use_qk_normalization=False, + rope_dim=rope_dim, + vision_encoder_in_channels=vision_encoder_in_channels, + fuse_qkv=fuse_qkv, + **model_arch_specs, + ) + # Vision-language tokenizer + tokenizer_config = TokenizerConfig( + text_tokenizer=TextTokenizerConfig( + config=L(ImageTextTokenizer)( + model_family=model_family, + is_instruct_model=is_instruct_model, + image_processor_path=tokenizer_ckpt_path, + tokenizer_path=tokenizer_ckpt_path, + ), + data_key="image_text_interleaved", + tokenizer_offset=model_config.vocab_size, + tokenize_here=False, + vocab_size=model_config.vocab_size, + ), + seq_len=model_config.max_seq_len, + training_type="image_text_interleaved", + add_special_tokens=add_special_tokens, + ) + return model_config, tokenizer_config + + +def create_video2world_model_config( + model_ckpt_path: str, + tokenizer_ckpt_path: str, + model_family: str = "cosmos", + model_size: str = "4b", + pixel_chunk_duration: int = 9, + num_video_frames: int = 36, + compression_ratio: List[int] = [8, 16, 16], + original_seq_len: int = 8192, + num_condition_latents_t: int = 1, + num_tokens_to_ignore: int = -1, + batch_size: int = 2, + video_tokenizer_config_creator: Callable = create_discrete_video_fsq_tokenizer_state_dict_config, + rope_dim: str = "3D", + add_special_tokens: bool = True, + video_height: int = 384, + video_width: int = 640, + use_qk_normalization: bool = True, + insert_cross_attn: bool = False, + insert_cross_attn_every_k_layers: int = 1, + context_dim: int = 1024, + training_type: str = "video_to_video", + pad_to_multiple_of: Optional[int] = 64, + vocab_size: int = 64000, + apply_abs_pos_emb: bool = False, +) -> dict: + """Create a video-to-world model config. + Args: + model_family (str): Model family. Choices: "llama", "llama3", "llama3.1", "mistral". + model_size (str): Model size. Choices: "1b", "8b", "3b". + pixel_chunk_duration (int): Number of frames in each chunk. + num_video_frames (int): Number of video frames. + compression_ratio (List[int]): Compression ratio for the video frames. Choices: [8, 16, 16] or [4, 8, 8]. + original_seq_len (int): Original sequence length. + apply_yarn (bool): Whether to apply YaRN for long context scaling. + yarn_beta_fast (Optional[int]): Fast beta for YaRN. + yarn_beta_slow (Optional[int]): Slow beta for YaRN. + yarn_scale (Optional[int]): Scale factor for ctx extension. + use_qk_normalization (bool): Whether to use Query-Key normalization. + training_type (str): Type of training task. + batch_size (int): Batch size. + video_tokenizer_config_creator (Callable): Method that takes "pixel_chunk_duration: int" and "version: str" as arguments and returns video tokenizer config + video_tokenizer_version (str): Version of the video tokenizer. + num_condition_latents_t (int): Number of conditioning latent channels + num_tokens_to_ignore (int) = Number of tokens to ignore. This takes the precedence + video_height (int): Height of the video frame. Defaults to 384. + video_width (int): Width of the video frame. Defaults to 640. + rope_dim (str): RoPE dimension. Choices: "1D", "3D". + add_special_tokens (bool): Whether to add special tokens, use False for 2D/3D RoPE. + pad_to_multiple_of (int): Pad the token sequence length to the nearest multiple of this number. Defaults to 64. + vocab_size (int): Vocabulary size. + apply_abs_pos_emb (bool): Whether to apply absolute positional embeddings. + Returns: + dict: A dictionary containing the model configuration representing the model object, can be instantiated. + """ + assert pixel_chunk_duration % compression_ratio[0] == 1, ( + f"pixel_chunk_duration({pixel_chunk_duration}) should be k*n + 1 (k={compression_ratio[0]})" + ) + latent_chunk_duration = (pixel_chunk_duration - 1) // compression_ratio[0] + 1 + latent_height = video_height // compression_ratio[1] + latent_width = video_width // compression_ratio[2] + # Do some math to compute the video latent shape and sequence length + assert num_video_frames % pixel_chunk_duration == 0, ( + f"num_video_frames {num_video_frames} should be divisible by pixel_chunk_duration {pixel_chunk_duration}" + ) + video_latent_shape = [ + num_video_frames // pixel_chunk_duration * latent_chunk_duration, + latent_height, + latent_width, + ] + # product of video_latent_shape + num_token_video_latent = video_latent_shape[0] * video_latent_shape[1] * video_latent_shape[2] + if add_special_tokens: + seq_len = num_token_video_latent + 3 # Sequence length per batch, max_seq_len + 3 + seq_len = (seq_len + 63) // 64 * 64 # Round up to multiple of 64 + # for text to video, we need to add token to indicate the start of the video + elif training_type == "text_to_video": + seq_len = num_token_video_latent + 1 + else: + seq_len = num_token_video_latent + + if seq_len % pad_to_multiple_of != 0: + # Round up to the nearest multiple of pad_to_multiple_of + seq_len = ((seq_len + pad_to_multiple_of - 1) // pad_to_multiple_of) * pad_to_multiple_of + + # Model size specific parameters + model_arch_specs = get_model_arch_specs(model_family=model_family, model_size=model_size, pretrained=True) + + # Whether skip the loss for first chunk or not, note the first token is already skipped when computing the loss + # If num_tokens_to_ignore is specified, use it. + # Else compute it from num_condition_latents_t + if num_tokens_to_ignore < 0: + num_tokens_to_ignore = latent_height * latent_width * num_condition_latents_t + if not add_special_tokens and num_condition_latents_t > 0: + # If there are no special tokens (bov), do a -1 so that you can compute the loss + # from the first token of the next chunk + num_tokens_to_ignore -= 1 + + model_config = ModelConfig( + video_height=video_height, + video_width=video_width, + max_seq_len=seq_len, + max_batch_size=batch_size, + precision="bfloat16", + ckpt_path=model_ckpt_path, + use_qk_normalization=use_qk_normalization, + vocab_size=64000, + original_seq_len=original_seq_len, + video_latent_shape=video_latent_shape, + num_video_frames=num_video_frames, + rope_dim=rope_dim, + pad_to_multiple_of=pad_to_multiple_of, + insert_cross_attn=insert_cross_attn, + insert_cross_attn_every_k_layers=insert_cross_attn_every_k_layers, + context_dim=context_dim, + apply_abs_pos_emb=apply_abs_pos_emb, + **model_arch_specs, + ) + + video_tokenizer_config = video_tokenizer_config_creator( + tokenizer_ckpt_path, pixel_chunk_duration, compression_ratio + ) + tokenizer_config = TokenizerConfig( + text_tokenizer=None, + video_tokenizer=VideoTokenizerConfig( + config=video_tokenizer_config, + data_key="video", + tokenizer_offset=0, # Since there is no text embeddings in the model. Note this only apply when the model is trained from scratch. If we use text pretrained model, the offset will be vocab_size of text token. + tokenize_here=True, + max_seq_len=num_token_video_latent, + vocab_size=vocab_size, + ), + seq_len=seq_len, + training_type=training_type, + add_special_tokens=add_special_tokens, + pad_to_multiple_of=pad_to_multiple_of, + ) + return model_config, tokenizer_config diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/tokenizer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/tokenizer.py new file mode 100644 index 00000000..17843412 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/base/tokenizer.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0116,C0301 + +from typing import Optional + +import attrs + +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQStateDictTokenizer +from cosmos1.models.autoregressive.tokenizer.networks import CausalDiscreteVideoTokenizer +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict + + +def create_discrete_video_fsq_tokenizer_state_dict_config( + ckpt_path, pixel_chunk_duration=33, compression_ratio=[8, 16, 16] +) -> LazyDict: + CausalDiscreteFactorizedVideoTokenizerConfig: LazyDict = L(CausalDiscreteVideoTokenizer)( + # The new causal discrete tokenizer, that is at least 2x more efficient in memory and runtime. + # - It relies on fully 3D discrete wavelet transform + # - Uses a layer norm instead of a group norm + # - Factorizes full convolutions into spatial and temporal convolutions + # - Factorizes full attention into spatial and temporal attention + # - Strictly causal, with flexible temporal length at inference. + attn_resolutions=[32], + channels=128, + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + z_channels=16, + z_factor=1, + num_groups=1, + legacy_mode=False, + spatial_compression=16, + temporal_compression=8, + embedding_dim=6, + levels=[8, 8, 8, 5, 5, 5], + name="CausalDiscreteFactorizedVideoTokenizer", + ) + + return L(DiscreteVideoFSQStateDictTokenizer)( + enc_fp=ckpt_path.replace("ema.jit", "encoder.jit"), + dec_fp=ckpt_path.replace("ema.jit", "decoder.jit"), + tokenizer_module=CausalDiscreteFactorizedVideoTokenizerConfig, + name="discrete_video_fsq", + latent_ch=6, + is_bf16=True, + pixel_chunk_duration=pixel_chunk_duration, + latent_chunk_duration=1 + (pixel_chunk_duration - 1) // compression_ratio[0], + max_enc_batch_size=8, + max_dec_batch_size=4, + levels=[8, 8, 8, 5, 5, 5], + compression_ratio=compression_ratio, + ) + + +@attrs.define(slots=False) +class TextTokenizerConfig: + """ + Text tokenizer config + + Args: + config: Config file to define the text tokenizer class. + data_key (str): The input key from data_dict that will be passed to the text tokenizer. + tokenize_here (bool): Whether to use the tokenizer to perform online tokenization. + tokenizer_offset (int): Offset that is added to the tokens. + vocab_size (int): Vocabulary size of the tokenizer. + """ + + config: LazyDict + data_key: str = "" + tokenize_here: bool = False + tokenizer_offset: int = 0 + vocab_size: int = 0 + + +@attrs.define(slots=False) +class VideoTokenizerConfig: + """ + Video tokenizer config + + Args: + config: Config file to define the video tokenizer class. + data_key (str): The input key from data_dict that will be passed to the video tokenizer. + tokenize_here (bool): Whether to use the tokenizer to perform online tokenization. + tokenizer_offset (int): Offset that is added to the tokens. In case of joint text-video tokenizers, we + add an offset to make sure that video tokens and text tokens don't overlap. + vocab_size (int): Vocabulary size of the tokenizer. + max_seq_len (int): Maximum token length for an input video. + """ + + config: LazyDict + data_key: str = "" + tokenize_here: bool = True + tokenizer_offset: int = 0 + vocab_size: int = 0 + max_seq_len: int = -1 + + +@attrs.define(slots=False) +class TokenizerConfig: + """ + Joint tokenizer config + + Args: + text_tokenizer (TextTokenizerConfig): Text tokenizer config file + class_tokenizer (ClassTokenizerConfig): Class tokenizer config file + video_tokenizer (VideoTokenizerConfig): Video tokenizer config file + image_tokenizer (ImageTokenizerConfig): Image tokenizer config file + seq_len (int): Final token sequence length + training_type (str): Type of training we use. Supports ["text_only", "text_to_video", "class_to_image", "image_text_interleaved"] + add_special_tokens (bool): Whether to add special tokens to the output tokens + pad_to_multiple_of (int): Pad the token sequence length to the nearest multiple of this number. Defaults to 64. + """ + + text_tokenizer: Optional[TextTokenizerConfig] = None + video_tokenizer: Optional[VideoTokenizerConfig] = None + seq_len: int = 4096 + training_type: str = None + add_special_tokens: bool = True + pad_to_multiple_of: Optional[int] = 64 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/inference/inference_config.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/inference/inference_config.py new file mode 100644 index 00000000..65ca8e8a --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/configs/inference/inference_config.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0301 + +from typing import Any, List, Union + +import attrs +from cosmos1.models.autoregressive.configs.base.model import ModelConfig, TokenizerConfig + + +@attrs.define(slots=False) +class DataShapeConfig: + latent_shape: list = [] + num_video_frames: Union[None, int] = None + height: Union[None, int] = None + width: Union[None, int] = None + + +@attrs.define(slots=False) +class SamplingConfig: + """ + Sampling config + Args: + temperature (float): Temperature value for controlling randomness in sampling. Defaults to 0.6. + top_p (float): Top-p probability threshold for nucleus sampling. Defaults to 0.9. + logprobs (bool): Flag indicating whether to compute token log probabilities. Defaults to False. + echo (bool): Flag indicating whether to include prompt tokens in the generated output. Defaults to False. + + """ + + temperature: float = 0.6 + top_k: int = None + top_p: float = 0.9 + compile_prefill: bool = False + compile_sampling: bool = True + logprobs: bool = False + echo: bool = False + + +@attrs.define(slots=False) +class DiffusionDecoderSamplingConfig: + """ + Diffusion decoder sampling config + Args: + guidance (float): Guidance scale for the diffusion process. Controls how much the model follows the conditioning. Defaults to 0.8. + sigma_min (float): Minimum noise level for the diffusion process. Defaults to 0.02. + sigma (float): Initial noise level for the diffusion process. Defaults to 8. + num_steps (int): Number of denoising steps to perform. Defaults to 35. + overlap (int): Number of overlapping frames between video chunks during processing. Defaults to 2. + continuous_tokenizer_channel (int): Number of channels in the continuous tokenizer of diffusion decoder. Defaults to 16. + continuous_tokenizer_spatial_compression_ratio (int): Spatial compression ratio for the continuous tokenizer of diffusion decoder. Defaults to 8. + dd_train_num_video_frames (int): Number of video frames used during training for diffusion decoder. Defaults to 57. + """ + + guidance: float = 1.8 + sigma_min: float = 0.02 + sigma: float = 8 + num_steps: int = 15 + overlap: int = 2 + continuous_tokenizer_channel = 16 + continuous_tokenizer_spatial_compression_ratio = 8 + dd_train_num_video_frames: int = 57 + max_iter: int = 99 + fps: int = 24 + + +@attrs.define(slots=False) +class InferenceConfig: + """ + Inference config + Args: + model_config (ModelConfig): Model config + tokenizer_config (TokenizerConfig): Tokenizer config + ckpt_path (str): Path to the checkpoint + latent_shape (list): Shape of the latent + """ + + model_config: ModelConfig = None + tokenizer_config: TokenizerConfig = None + ckpt_path: str = "" + data_shape_config: DataShapeConfig = None + + defaults: List[Any] = attrs.field( + factory=lambda: [ + "_self_", + {"data_val": None}, + {"data_shape_config": "video_shape_as_model_config"}, + {"eval_job": None}, + ] + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/base/conditioner.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/base/conditioner.py new file mode 100644 index 00000000..7a8d57fd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/base/conditioner.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from dataclasses import dataclass +from typing import Dict, Optional + +import torch +from cosmos1.models.diffusion.conditioner import BaseVideoCondition, GeneralConditioner +from cosmos1.models.diffusion.config.base.conditioner import ( + FPSConfig, + ImageSizeConfig, + LatentConditionConfig, + LatentConditionSigmaConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, +) +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict + + +@dataclass +class VideoLatentDiffusionDecoderCondition(BaseVideoCondition): + # latent_condition will concat to the input of network, along channel dim; + # cfg will make latent_condition all zero padding. + latent_condition: Optional[torch.Tensor] = None + latent_condition_sigma: Optional[torch.Tensor] = None + + +class VideoDiffusionDecoderConditioner(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> VideoLatentDiffusionDecoderCondition: + output = super()._forward(batch, override_dropout_rate) + return VideoLatentDiffusionDecoderCondition(**output) + + +VideoLatentDiffusionDecoderConditionerConfig: LazyDict = L(VideoDiffusionDecoderConditioner)( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + latent_condition=LatentConditionConfig(), + latent_condition_sigma=LatentConditionSigmaConfig(), +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/config_latent_diffusion_decoder.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/config_latent_diffusion_decoder.py new file mode 100644 index 00000000..4134e469 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/config_latent_diffusion_decoder.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116 + +from typing import Any, List + +import attrs +from cosmos1.models.autoregressive.diffusion_decoder.config.registry import register_configs as register_dd_configs +from cosmos1.models.diffusion.config.base.model import LatentDiffusionDecoderModelConfig +from cosmos1.models.diffusion.config.registry import register_configs +from cosmos1.utils import config +from cosmos1.utils.config_helper import import_all_modules_from_package + + +@attrs.define(slots=False) +class Config(config.Config): + # default config groups that will be used unless overwritten + # see config groups in registry.py + defaults: List[Any] = attrs.field( + factory=lambda: [ + "_self_", + {"net": None}, + {"conditioner": "basic"}, + {"tokenizer": "tokenizer"}, + {"tokenizer_corruptor": None}, + {"latent_corruptor": None}, + {"pixel_corruptor": None}, + {"experiment": None}, + ] + ) + + +def make_config(): + c = Config(model=LatentDiffusionDecoderModelConfig()) + + # Specifying values through instances of attrs + c.job.project = "cosmos_video4" + c.job.group = "debug" + c.job.name = "delete_${now:%Y-%m-%d}_${now:%H-%M-%S}" + + # Call this function to register config groups for advanced overriding. + register_configs() + register_dd_configs() + + # experiment config are defined in the experiment folder + # call import_all_modules_from_package to register them + import_all_modules_from_package("cosmos1.models.diffusion.config.inference", reload=True) + import_all_modules_from_package("cosmos1.models.autoregressive.diffusion_decoder.config.inference", reload=True) + return c diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/inference/cosmos_diffusiondecoder_7b.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/inference/cosmos_diffusiondecoder_7b.py new file mode 100644 index 00000000..48d887af --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/inference/cosmos_diffusiondecoder_7b.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cosmos1.models.autoregressive.diffusion_decoder.network import DiffusionDecoderGeneralDIT +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict +from hydra.core.config_store import ConfigStore + + +num_frames = 57 +Cosmos_DiffusionDecoder_7B_INFERENCE_ONLY: LazyDict = LazyDict( + dict( + defaults=[ + {"override /net": "faditv2_7b"}, + {"override /tokenizer": "cosmos_video_tokenizer_res720_comp8x8x8_t121_ver092624"}, + {"override /conditioner": "video_latent_diffusion_decoder_cond"}, + {"override /tokenizer_corruptor": "cosmos_video_discrete_tokenizer_res720_comp8x16x16_t49_ver110224"}, + "_self_", + ], + job=dict( + group="diffusion_deocder_FT_7Bv1_001", + name="DD_FT_7Bv1_003_002_tokenizer888_spatch2_discrete_cond_on_token", + ), + model=dict( + diffusion_decoder_cond_sigma_low=0.0, + diffusion_decoder_cond_sigma_high=0.0, + diffusion_decoder_corrupt_prob=0.0, + condition_on_tokenizer_corruptor_token=True, + latent_shape=[ + 16, + num_frames, + 88, + 160, + ], + tokenizer_corruptor=dict( + pixel_chunk_duration=num_frames, + latent_chunk_duration=1 + (num_frames - 1) // 8, + ), + net=L(DiffusionDecoderGeneralDIT)( + diffusion_decoder_condition_on_sigma=False, + max_img_h=240, + max_img_w=240, + rope_h_extrapolation_ratio=1.5, + rope_w_extrapolation_ratio=1.5, + rope_t_extrapolation_ratio=1, + block_x_format="THWBD", + is_diffusion_decoder=True, + patch_spatial=2, + diffusion_decoder_condition_on_token=True, + diffusion_decoder_token_condition_voc_size=64000, + diffusion_decoder_token_condition_dim=32, + ), + tokenizer=dict( + video_vae=dict( + pixel_chunk_duration=num_frames, + ) + ), + conditioner=dict( + latent_condition=dict( + dropout_rate=0.2, + ) + ), + ), + ) +) + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_DiffusionDecoder_7B_INFERENCE_ONLY["job"]["name"], + node=Cosmos_DiffusionDecoder_7B_INFERENCE_ONLY, +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/registry.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/registry.py new file mode 100644 index 00000000..cd8b7c0f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/config/registry.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116 + +from cosmos1.models.autoregressive.diffusion_decoder.config.base.conditioner import ( + VideoLatentDiffusionDecoderConditionerConfig, +) +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQJITTokenizer +from cosmos1.models.diffusion.module.pretrained_vae import JITVAE, JointImageVideoSharedJITTokenizer, VideoJITTokenizer +from cosmos1.utils.lazy_config import LazyCall as L +from hydra.core.config_store import ConfigStore + + +def get_cosmos_video_discrete_tokenizer_comp8x16x16( + resolution: str, + chunk_duration: int, + checkpoint_path: str, +): + assert resolution in ["720"] + + pixel_chunk_duration = chunk_duration + temporal_compression_factor = 8 + spatial_compression_factor = 16 + + return L(DiscreteVideoFSQJITTokenizer)( + enc_fp=checkpoint_path.replace(".jit", "encoder.jit"), + dec_fp=checkpoint_path.replace(".jit", "decoder.jit"), + name="discrete_video_fsq", + latent_ch=6, + is_bf16=True, + pixel_chunk_duration=pixel_chunk_duration, + latent_chunk_duration=1 + (pixel_chunk_duration - 1) // temporal_compression_factor, + max_enc_batch_size=8, + max_dec_batch_size=4, + levels=[8, 8, 8, 5, 5, 5], + compression_ratio=[temporal_compression_factor, spatial_compression_factor, spatial_compression_factor], + ) + + +def get_cosmos_video_tokenizer_comp8x8x8(resolution: str, chunk_duration: int, checkpoint_path=None): + pixel_chunk_duration = chunk_duration + temporal_compression_factor = 8 + spatial_compression_factor = 8 + + return L(JointImageVideoSharedJITTokenizer)( + video_vae=L(VideoJITTokenizer)( + name="cosmos_1_0_diffusion_tokenizer", + latent_ch=16, + is_bf16=True, + pixel_chunk_duration=pixel_chunk_duration, + temporal_compression_factor=temporal_compression_factor, + spatial_compression_factor=spatial_compression_factor, + spatial_resolution=resolution, + ), + image_vae=L(JITVAE)( + name="cosmos_1_0_diffusion_tokenizer", + latent_ch=16, + is_image=False, + is_bf16=True, + ), + name="cosmos_diffusion_tokenizer_res720_comp8x8x8_t121_ver092624", + latent_ch=16, + ) + + +def register_tokenizer(cs): + cs.store( + group="tokenizer", + package="model.tokenizer", + name="cosmos_video_tokenizer_res720_comp8x8x8_t121_ver092624", + node=get_cosmos_video_tokenizer_comp8x8x8( + resolution="720", + chunk_duration=121, + checkpoint_path="checkpoints/Cosmos-1.0-Tokenizer-CV8x8x8/.jit", + ), + ) + + +def register_corruptor(cs): + cs.store( + group="tokenizer_corruptor", + package="model.tokenizer_corruptor", + name="cosmos_video_discrete_tokenizer_res720_comp8x16x16_t49_ver110224", + node=get_cosmos_video_discrete_tokenizer_comp8x16x16( + resolution="720", + chunk_duration=49, + checkpoint_path="checkpoints/Cosmos-1.0-Tokenizer-DV8x16x16/.jit", + ), + ) + + +def register_conditioner(cs): + cs.store( + group="conditioner", + package="model.conditioner", + name="video_latent_diffusion_decoder_cond", + node=VideoLatentDiffusionDecoderConditionerConfig, + ) + + +def register_configs(): + cs = ConfigStore.instance() + + register_conditioner(cs) + register_corruptor(cs) + register_tokenizer(cs) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/inference.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/inference.py new file mode 100644 index 00000000..f157219c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/inference.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0116 + +import copy +import gc +from typing import List + +import torch + +from cosmos1.models.autoregressive.configs.inference.inference_config import DiffusionDecoderSamplingConfig +from cosmos1.models.autoregressive.diffusion_decoder.model import LatentDiffusionDecoderModel +from cosmos1.models.autoregressive.diffusion_decoder.utils import linear_blend_video_list, split_with_overlap +from cosmos1.utils import log + + +def diffusion_decoder_process_tokens( + model: LatentDiffusionDecoderModel, + indices_tensor: List[torch.Tensor], + dd_sampling_config: DiffusionDecoderSamplingConfig = None, + original_video_example: torch.Tensor = None, + t5_emb_batch: List[torch.Tensor] = None, +): + _, T, H, W = original_video_example.shape + if dd_sampling_config is None: + dd_sampling_config = DiffusionDecoderSamplingConfig() + # indices_tensor is assumed to be a list of tensors with shape 1LHW + data_batch_list = [] + for sample_num, token_CTHW in enumerate(indices_tensor): + token_BCTHW = token_CTHW.unsqueeze(0).unsqueeze(1) + token_BCTHW = split_with_overlap( + token_BCTHW, + (dd_sampling_config.dd_train_num_video_frames - 1) // 8 + 1, + overlap=dd_sampling_config.overlap, + tobf16=False, + ) + data_batch_list.append( + { + "token_chunks": token_BCTHW, + "t5_text_embeddings": t5_emb_batch[sample_num].to(torch.bfloat16), + "t5_text_mask": torch.ones(1, 512, dtype=torch.bfloat16).cuda(), + # other conditions + "image_size": torch.tensor([[H, W, H, W]] * 1, dtype=torch.bfloat16).cuda(), + "fps": torch.tensor([dd_sampling_config.fps] * 1, dtype=torch.bfloat16).cuda(), + "num_frames": torch.tensor( + [dd_sampling_config.dd_train_num_video_frames] * 1, dtype=torch.bfloat16 + ).cuda(), + "padding_mask": torch.zeros((1, 1, H, W), dtype=torch.bfloat16).cuda(), + } + ) + + out_videos_batch = [] + + for idx, data_batch_template in enumerate(data_batch_list): + full_length_sample = [] + iterations = min(len(data_batch_template["token_chunks"]), dd_sampling_config.max_iter) + for iter in range(iterations): + gc.collect() + torch.cuda.empty_cache() + + data_batch = copy.deepcopy(data_batch_template) + data_batch["video"] = data_batch_template["token_chunks"][iter].cuda().to("cuda") + + log.debug(f"Run iter {iter} for video # {idx} at length {data_batch['video'].shape[2]}") + # org_video, + with torch.no_grad(): + samples_latent = model.generate_samples_from_batch( + data_batch, + guidance=dd_sampling_config.guidance, + sigma_min=dd_sampling_config.sigma_min, + state_shape=[ + dd_sampling_config.continuous_tokenizer_channel, + dd_sampling_config.continuous_tokenizer_spatial_compression_ratio, + H // 8, + W // 8, + ], + apply_corruptor=False, + return_recon_x=False, + # corrupt_sigma=dd_sampling_config.sigma, + preencode_condition=True, # We are using discrete model, so the input is already pre-encoded + num_steps=dd_sampling_config.num_steps, + ) + log.debug(f"Current sample shape {samples_latent.shape} for video # {idx} ") + full_length_sample.append(samples_latent.detach()) + + # Turn off because we remove CP + # distributed.barrier() + del data_batch + + torch.cuda.empty_cache() + + gc.collect() + torch.cuda.empty_cache() + + # Decode full-length samples and free GPU memory + full_length_sample_pixs = [model.decode(item).clamp(-1, 1).cpu() for item in full_length_sample] + torch.cuda.empty_cache() + + # Blend pixel samples + if len(full_length_sample_pixs) > 1: + full_length_sample_pixel_blend = linear_blend_video_list( + full_length_sample_pixs, dd_sampling_config.overlap + )[:, :, :T] + else: + full_length_sample_pixel_blend = full_length_sample_pixs[0][:, :, :T] + + # Batch size of full_length_sample_pixel_blend is always 1 + out_videos_batch.append((1 + full_length_sample_pixel_blend[0].cpu()) / 2) + return out_videos_batch diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/model.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/model.py new file mode 100644 index 00000000..5e71a8e4 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/model.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0301 + +from dataclasses import dataclass +from typing import Callable, Dict, Optional, Tuple + +import torch +from torch import Tensor + +from cosmos1.models.diffusion.conditioner import BaseVideoCondition +from cosmos1.models.diffusion.diffusion.functional.batch_ops import batch_mul +from cosmos1.models.diffusion.diffusion.modules.res_sampler import COMMON_SOLVER_OPTIONS +from cosmos1.models.diffusion.model.model_t2w import DiffusionT2WModel as VideoDiffusionModel +from cosmos1.utils.lazy_config import instantiate as lazy_instantiate + + +@dataclass +class VideoLatentDiffusionDecoderCondition(BaseVideoCondition): + # latent_condition will concat to the input of network, along channel dim; + # cfg will make latent_condition all zero padding. + latent_condition: Optional[torch.Tensor] = None + latent_condition_sigma: Optional[torch.Tensor] = None + + +class LatentDiffusionDecoderModel(VideoDiffusionModel): + def __init__(self, config): + super().__init__(config) + """ + latent_corruptor: the corruption module is used to corrupt the latents. It add gaussian noise to the latents. + pixel_corruptor: the corruption module is used to corrupt the pixels. It apply gaussian blur kernel to pixels in a temporal consistent way. + tokenizer_corruptor: the corruption module is used to simulate tokenizer reconstruction errors. + + diffusion decoder noise augmentation pipeline for continuous token condition model: + condition: GT_video [T, H, W] + -> tokenizer_corruptor~(8x8x8) encode -> latent_corruptor -> tokenizer_corruptor~(8x8x8) decode + -> pixel corruptor + -> tokenizer~(1x8x8) encode -> condition [T, H/8, W/8] + GT: GT_video [T, H, W] -> tokenizer~(1x8x8) -> x_t [T, H/8, W/8]. + + diffusion decoder noise augmentation pipeline for discrete token condition model: + condition: GT_video [T, H, W] + -> pixel corruptor + -> discrete tokenizer encode -> condition [T, T/8, H/16, W/16] + GT: GT_video [T, H, W] -> tokenizer~(8x8x8) -> x_t [T, T/8, H/8, W/8]. + + """ + self.latent_corruptor = lazy_instantiate(config.latent_corruptor) + self.pixel_corruptor = lazy_instantiate(config.pixel_corruptor) + self.tokenizer_corruptor = lazy_instantiate(config.tokenizer_corruptor) + + if self.latent_corruptor: + self.latent_corruptor.to(**self.tensor_kwargs) + if self.pixel_corruptor: + self.pixel_corruptor.to(**self.tensor_kwargs) + + if self.tokenizer_corruptor: + if hasattr(self.tokenizer_corruptor, "reset_dtype"): + self.tokenizer_corruptor.reset_dtype() + else: + assert self.pixel_corruptor is not None + + self.diffusion_decoder_cond_sigma_low = config.diffusion_decoder_cond_sigma_low + self.diffusion_decoder_cond_sigma_high = config.diffusion_decoder_cond_sigma_high + self.diffusion_decoder_corrupt_prob = config.diffusion_decoder_corrupt_prob + if hasattr(config, "condition_on_tokenizer_corruptor_token"): + self.condition_on_tokenizer_corruptor_token = config.condition_on_tokenizer_corruptor_token + else: + self.condition_on_tokenizer_corruptor_token = False + + def is_image_batch(self, data_batch: dict[str, Tensor]) -> bool: + """We hanlde two types of data_batch. One comes from a joint_dataloader where "dataset_name" can be used to differenciate image_batch and video_batch. + Another comes from a dataloader which we by default assumes as video_data for video model training. + """ + is_image = self.input_image_key in data_batch + is_video = self.input_data_key in data_batch + assert is_image != is_video, ( + "Only one of the input_image_key or input_data_key should be present in the data_batch." + ) + return is_image + + def get_x0_fn_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + apply_corruptor: bool = True, + corrupt_sigma: float = 1.5, + preencode_condition: bool = False, + ) -> Callable: + """ + Generates a callable function `x0_fn` based on the provided data batch and guidance factor. + + This function first processes the input data batch through a conditioning workflow (`conditioner`) to obtain conditioned and unconditioned states. It then defines a nested function `x0_fn` which applies a denoising operation on an input `noise_x` at a given noise level `sigma` using both the conditioned and unconditioned states. + + Args: + - data_batch (Dict): A batch of data used for conditioning. The format and content of this dictionary should align with the expectations of the `self.conditioner` + - guidance (float, optional): A scalar value that modulates the influence of the conditioned state relative to the unconditioned state in the output. Defaults to 1.5. + - is_negative_prompt (bool): use negative prompt t5 in uncondition if true + + Returns: + - Callable: A function `x0_fn(noise_x, sigma)` that takes two arguments, `noise_x` and `sigma`, and return x0 predictoin + + The returned function is suitable for use in scenarios where a denoised state is required based on both conditioned and unconditioned inputs, with an adjustable level of guidance influence. + """ + input_key = self.input_data_key # by default it is video key + # Latent state + raw_state = data_batch[input_key] + + if self.condition_on_tokenizer_corruptor_token: + if preencode_condition: + latent_condition = raw_state.to(torch.int32).contiguous() + corrupted_pixel = self.tokenizer_corruptor.decode(latent_condition[:, 0]) + else: + corrupted_pixel = ( + self.pixel_corruptor(raw_state) if apply_corruptor and self.pixel_corruptor else raw_state + ) + latent_condition = self.tokenizer_corruptor.encode(corrupted_pixel) + latent_condition = latent_condition[1] if isinstance(latent_condition, tuple) else latent_condition + corrupted_pixel = self.tokenizer_corruptor.decode(latent_condition) + latent_condition = latent_condition.unsqueeze(1) + else: + if preencode_condition: + latent_condition = raw_state + corrupted_pixel = self.decode(latent_condition) + else: + corrupted_pixel = ( + self.pixel_corruptor(raw_state) if apply_corruptor and self.pixel_corruptor else raw_state + ) + latent_condition = self.encode(corrupted_pixel).contiguous() + + sigma = ( + torch.rand((latent_condition.shape[0],)).to(**self.tensor_kwargs) * corrupt_sigma + ) # small value to indicate clean video + _, _, _, c_noise_cond = self.scaling(sigma=sigma) + if corrupt_sigma != self.diffusion_decoder_cond_sigma_low and self.diffusion_decoder_corrupt_prob > 0: + noise = batch_mul(sigma, torch.randn_like(latent_condition)) + latent_condition = latent_condition + noise + data_batch["latent_condition_sigma"] = batch_mul(torch.ones_like(latent_condition[:, 0:1, ::]), c_noise_cond) + data_batch["latent_condition"] = latent_condition + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + cond_x0 = self.denoise(noise_x, sigma, condition).x0 + uncond_x0 = self.denoise(noise_x, sigma, uncondition).x0 + return cond_x0 + guidance * (cond_x0 - uncond_x0) + + return x0_fn, corrupted_pixel + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + solver_option: COMMON_SOLVER_OPTIONS = "2ab", + sigma_min: float = 0.02, + apply_corruptor: bool = False, + return_recon_x: bool = False, + corrupt_sigma: float = 0.01, + preencode_condition: bool = False, + ) -> Tensor: + """ + Generate samples from the batch. Based on given batch, it will automatically determine whether to generate image or video samples. + Args: + data_batch (dict): raw data batch draw from the training data loader. + iteration (int): Current iteration number. + guidance (float): guidance weights + seed (int): random seed + state_shape (tuple): shape of the state, default to self.state_shape if not provided + n_sample (int): number of samples to generate + is_negative_prompt (bool): use negative prompt t5 in uncondition if true + num_steps (int): number of steps for the diffusion process + solver_option (str): differential equation solver option, default to "2ab"~(mulitstep solver) + preencode_condition (bool): use pre-computed condition if true, save tokenizer's inference time memory/ + """ + if not preencode_condition: + self._normalize_video_databatch_inplace(data_batch) + self._augment_image_dim_inplace(data_batch) + is_image_batch = False + if n_sample is None: + input_key = self.input_image_key if is_image_batch else self.input_data_key + n_sample = data_batch[input_key].shape[0] + if state_shape is None: + if is_image_batch: + state_shape = (self.state_shape[0], 1, *self.state_shape[2:]) # C,T,H,W + + x0_fn, recon_x = self.get_x0_fn_from_batch( + data_batch, + guidance, + is_negative_prompt=is_negative_prompt, + apply_corruptor=apply_corruptor, + corrupt_sigma=corrupt_sigma, + preencode_condition=preencode_condition, + ) + generator = torch.Generator(device=self.tensor_kwargs["device"]) + generator.manual_seed(seed) + x_sigma_max = ( + torch.randn(n_sample, *state_shape, **self.tensor_kwargs, generator=generator) * self.sde.sigma_max + ) + + samples = self.sampler( + x0_fn, + x_sigma_max, + num_steps=num_steps, + sigma_min=sigma_min, + sigma_max=self.sde.sigma_max, + solver_option=solver_option, + ) + + if return_recon_x: + return samples, recon_x + else: + return samples diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/network.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/network.py new file mode 100644 index 00000000..db188239 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/network.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional, Tuple + +import torch +from einops import rearrange +from torch import nn +from torchvision import transforms + +from cosmos1.models.diffusion.module.blocks import PatchEmbed +from cosmos1.models.diffusion.networks.general_dit import GeneralDIT + + +class DiffusionDecoderGeneralDIT(GeneralDIT): + def __init__( + self, + *args, + is_diffusion_decoder: bool = True, + diffusion_decoder_condition_on_sigma: bool = False, + diffusion_decoder_condition_on_token: bool = False, + diffusion_decoder_token_condition_voc_size: int = 64000, + diffusion_decoder_token_condition_dim: int = 32, + **kwargs, + ): + # diffusion decoder setting + self.is_diffusion_decoder = is_diffusion_decoder + self.diffusion_decoder_condition_on_sigma = diffusion_decoder_condition_on_sigma + self.diffusion_decoder_condition_on_token = diffusion_decoder_condition_on_token + self.diffusion_decoder_token_condition_voc_size = diffusion_decoder_token_condition_voc_size + self.diffusion_decoder_token_condition_dim = diffusion_decoder_token_condition_dim + super().__init__(*args, **kwargs) + + def initialize_weights(self): + # Initialize transformer layers: + super().initialize_weights() + if self.diffusion_decoder_condition_on_token: + nn.init.constant_(self.token_embedder.weight, 0) + + def build_patch_embed(self): + ( + concat_padding_mask, + in_channels, + patch_spatial, + patch_temporal, + model_channels, + is_diffusion_decoder, + diffusion_decoder_condition_on_sigma, + ) = ( + self.concat_padding_mask, + self.in_channels, + self.patch_spatial, + self.patch_temporal, + self.model_channels, + self.is_diffusion_decoder, + self.diffusion_decoder_condition_on_sigma, + ) + in_channels = ( + in_channels + in_channels + if (is_diffusion_decoder and not self.diffusion_decoder_condition_on_token) + else in_channels + ) + in_channels = in_channels + 1 if diffusion_decoder_condition_on_sigma else in_channels + in_channels = ( + in_channels + self.diffusion_decoder_token_condition_dim + if self.diffusion_decoder_condition_on_token + else in_channels + ) + in_channels = in_channels + 1 if concat_padding_mask else in_channels + + self.x_embedder = PatchEmbed( + spatial_patch_size=patch_spatial, + temporal_patch_size=patch_temporal, + in_channels=in_channels, + out_channels=model_channels, + bias=False, + ) + + if self.diffusion_decoder_condition_on_token: + self.token_embedder = nn.Embedding( + self.diffusion_decoder_token_condition_voc_size, self.diffusion_decoder_token_condition_dim + ) + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Prepares an embedded sequence tensor by applying positional embeddings and handling padding masks. + + Args: + x_B_C_T_H_W (torch.Tensor): video + fps (Optional[torch.Tensor]): Frames per second tensor to be used for positional embedding when required. + If None, a default value (`self.base_fps`) will be used. + padding_mask (Optional[torch.Tensor]): current it is not used + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - A tensor of shape (B, T, H, W, D) with the embedded sequence. + - An optional positional embedding tensor, returned only if the positional embedding class + (`self.pos_emb_cls`) includes 'rope'. Otherwise, None. + + Notes: + - If `self.concat_padding_mask` is True, a padding mask channel is concatenated to the input tensor. + - The method of applying positional embeddings depends on the value of `self.pos_emb_cls`. + - If 'rope' is in `self.pos_emb_cls` (case insensitive), the positional embeddings are generated using + the `self.pos_embedder` with the shape [T, H, W]. + - If "fps_aware" is in `self.pos_emb_cls`, the positional embeddings are generated using the `self.pos_embedder` + with the fps tensor. + - Otherwise, the positional embeddings are generated without considering fps. + """ + if self.diffusion_decoder_condition_on_token: + latent_condition = self.token_embedder(latent_condition) + B, _, T, H, W, _ = latent_condition.shape + latent_condition = rearrange(latent_condition, "B 1 T H W D -> (B T) (1 D) H W") + + latent_condition = transforms.functional.resize( + latent_condition, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.BILINEAR + ) + latent_condition = rearrange(latent_condition, "(B T) D H W -> B D T H W ", B=B, T=T) + x_B_C_T_H_W = torch.cat([x_B_C_T_H_W, latent_condition], dim=1) + if self.diffusion_decoder_condition_on_sigma: + x_B_C_T_H_W = torch.cat([x_B_C_T_H_W, latent_condition_sigma], dim=1) + if self.concat_padding_mask: + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None, extra_pos_emb diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/utils.py new file mode 100644 index 00000000..2c584c7c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/diffusion_decoder/utils.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn.functional as F + + +def split_with_overlap(video_BCTHW, num_video_frames, overlap=2, tobf16=True): + """ + Splits the video tensor into chunks of num_video_frames with a specified overlap. + + Args: + - video_BCTHW (torch.Tensor): Input tensor with shape [Batch, Channels, Time, Height, Width]. + - num_video_frames (int): Number of frames per chunk. + - overlap (int): Number of overlapping frames between chunks. + + Returns: + - List of torch.Tensors: List of video chunks with overlap. + """ + # Get the dimensions of the input tensor + B, C, T, H, W = video_BCTHW.shape + + # Ensure overlap is less than num_video_frames + assert overlap < num_video_frames, "Overlap should be less than num_video_frames." + + # List to store the chunks + chunks = [] + + # Step size for the sliding window + step = num_video_frames - overlap + + # Loop through the time dimension (T) with the sliding window + for start in range(0, T - overlap, step): + end = start + num_video_frames + # Handle the case when the last chunk might go out of bounds + if end > T: + # Get the last available frame + num_padding_frames = end - T + chunk = F.pad(video_BCTHW[:, :, start:T, :, :], (0, 0, 0, 0, 0, num_padding_frames), mode="reflect") + else: + # Regular case: no padding needed + chunk = video_BCTHW[:, :, start:end, :, :] + if tobf16: + chunks.append(chunk.to(torch.bfloat16)) + else: + chunks.append(chunk) + return chunks + + +def linear_blend_video_list(videos, D): + """ + Linearly blends a list of videos along the time dimension with overlap length D. + + Parameters: + - videos: list of video tensors, each of shape [b, c, t, h, w] + - D: int, overlap length + + Returns: + - output_video: blended video tensor of shape [b, c, L, h, w] + """ + assert len(videos) >= 2, "At least two videos are required." + b, c, t, h, w = videos[0].shape + N = len(videos) + + # Ensure all videos have the same shape + for video in videos: + assert video.shape == (b, c, t, h, w), "All videos must have the same shape." + + # Calculate total output length + L = N * t - D * (N - 1) + output_video = torch.zeros((b, c, L, h, w), device=videos[0].device) + + output_index = 0 # Current index in the output video + + for i in range(N): + if i == 0: + # Copy frames from the first video up to t - D + output_video[:, :, output_index : output_index + t - D, :, :] = videos[i][:, :, : t - D, :, :] + output_index += t - D + else: + # Blend overlapping frames between videos[i-1] and videos[i] + blend_weights = torch.linspace(0, 1, steps=D, device=videos[0].device) + + for j in range(D): + w1 = 1 - blend_weights[j] + w2 = blend_weights[j] + frame_from_prev = videos[i - 1][:, :, t - D + j, :, :] + frame_from_curr = videos[i][:, :, j, :, :] + output_frame = w1 * frame_from_prev + w2 * frame_from_curr + output_video[:, :, output_index, :, :] = output_frame + output_index += 1 + + if i < N - 1: + # Copy non-overlapping frames from current video up to t - D + frames_to_copy = t - 2 * D + if frames_to_copy > 0: + output_video[:, :, output_index : output_index + frames_to_copy, :, :] = videos[i][ + :, :, D : t - D, :, : + ] + output_index += frames_to_copy + else: + # For the last video, copy frames from D to t + frames_to_copy = t - D + output_video[:, :, output_index : output_index + frames_to_copy, :, :] = videos[i][:, :, D:, :, :] + output_index += frames_to_copy + + return output_video diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/base.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/base.py new file mode 100644 index 00000000..d4275275 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/base.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import imageio +import torch + +from cosmos1.models.autoregressive.inference.world_generation_pipeline import ARBaseGenerationPipeline +from cosmos1.models.autoregressive.utils.inference import add_common_arguments, load_vision_input, validate_args +from cosmos1.utils import log + + +def parse_args(): + parser = argparse.ArgumentParser(description="Video to world generation demo script") + # Add common arguments + add_common_arguments(parser) + parser.add_argument( + "--ar_model_dir", + type=str, + default="Cosmos-1.0-Autoregressive-4B", + ) + parser.add_argument("--input_type", type=str, default="video", help="Type of input", choices=["image", "video"]) + args = parser.parse_args() + return args + + +def main(args): + """Run video-to-world generation demo. + + This function handles the main video-to-world generation pipeline, including: + - Setting up the random seed for reproducibility + - Initializing the generation pipeline with the provided configuration + - Processing single or multiple images/videos from input + - Generating videos from images/videos + - Saving the generated videos to disk + + Args: + cfg (argparse.Namespace): Configuration namespace containing: + - Model configuration (checkpoint paths, model settings) + - Generation parameters (temperature, top_p) + - Input/output settings (images/videos, save paths) + - Performance options (model offloading settings) + + The function will save: + - Generated MP4 video files + + If guardrails block the generation, a critical log message is displayed + and the function continues to the next prompt if available. + """ + inference_type = "base" # When the inference_type is "base", AR model does not take text as input, the world generation is purely based on the input video + sampling_config = validate_args(args, inference_type) + + # Initialize base generation model pipeline + pipeline = ARBaseGenerationPipeline( + inference_type=inference_type, + checkpoint_dir=args.checkpoint_dir, + checkpoint_name=args.ar_model_dir, + disable_diffusion_decoder=args.disable_diffusion_decoder, + offload_guardrail_models=args.offload_guardrail_models, + offload_diffusion_decoder=args.offload_diffusion_decoder, + offload_network=args.offload_ar_model, + offload_tokenizer=args.offload_tokenizer, + ) + + # Load input image(s) or video(s) + input_videos = load_vision_input( + input_type=args.input_type, + batch_input_path=args.batch_input_path, + input_image_or_video_path=args.input_image_or_video_path, + data_resolution=args.data_resolution, + num_input_frames=args.num_input_frames, + ) + + for idx, input_filename in enumerate(input_videos): + inp_vid = input_videos[input_filename] + # Generate video + log.info(f"Run with image or video path: {input_filename}") + out_vid = pipeline.generate( + inp_vid=inp_vid, + num_input_frames=args.num_input_frames, + seed=args.seed, + sampling_config=sampling_config, + ) + if out_vid is None: + log.critical("Guardrail blocked base generation.") + continue + + # Save video + if args.input_image_or_video_path: + out_vid_path = os.path.join(args.video_save_folder, f"{args.video_save_name}.mp4") + else: + out_vid_path = os.path.join(args.video_save_folder, f"{idx}.mp4") + + imageio.mimsave(out_vid_path, out_vid, fps=25) + + log.info(f"Saved video to {out_vid_path}") + + +if __name__ == "__main__": + torch._C._jit_set_texpr_fuser_enabled(False) + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/video2world.py new file mode 100644 index 00000000..4c5bf25d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/video2world.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import imageio +import torch + +from cosmos1.models.autoregressive.inference.world_generation_pipeline import ARVideo2WorldGenerationPipeline +from cosmos1.models.autoregressive.utils.inference import add_common_arguments, load_vision_input, validate_args +from cosmos1.utils import log +from cosmos1.utils.io import read_prompts_from_file + + +def parse_args(): + parser = argparse.ArgumentParser(description="Prompted video to world generation demo script") + add_common_arguments(parser) + parser.add_argument( + "--ar_model_dir", + type=str, + default="Cosmos-1.0-Autoregressive-5B-Video2World", + ) + parser.add_argument( + "--input_type", + type=str, + default="text_and_video", + choices=["text_and_image", "text_and_video"], + help="Input types", + ) + parser.add_argument( + "--prompt", + type=str, + help="Text prompt for generating a single video", + ) + parser.add_argument( + "--offload_text_encoder_model", + action="store_true", + help="Offload T5 model after inference", + ) + args = parser.parse_args() + return args + + +def main(args): + """Run prompted video-to-world generation demo. + + This function handles the main video-to-world generation pipeline, including: + - Setting up the random seed for reproducibility + - Initializing the generation pipeline with the provided configuration + - Processing single or multiple prompts/images/videos from input + - Generating videos from prompts and images/videos + - Saving the generated videos and corresponding prompts to disk + + Args: + cfg (argparse.Namespace): Configuration namespace containing: + - Model configuration (checkpoint paths, model settings) + - Generation parameters (temperature, top_p) + - Input/output settings (images/videos, save paths) + - Performance options (model offloading settings) + + The function will save: + - Generated MP4 video files + + If guardrails block the generation, a critical log message is displayed + and the function continues to the next prompt if available. + """ + inference_type = "video2world" # When the inference_type is "video2world", AR model takes both text and video as input, the world generation is based on the input text prompt and video + sampling_config = validate_args(args, inference_type) + + # Initialize prompted base generation model pipeline + pipeline = ARVideo2WorldGenerationPipeline( + inference_type=inference_type, + checkpoint_dir=args.checkpoint_dir, + checkpoint_name=args.ar_model_dir, + disable_diffusion_decoder=args.disable_diffusion_decoder, + offload_guardrail_models=args.offload_guardrail_models, + offload_diffusion_decoder=args.offload_diffusion_decoder, + offload_network=args.offload_ar_model, + offload_tokenizer=args.offload_tokenizer, + offload_text_encoder_model=args.offload_text_encoder_model, + ) + + # Load input image(s) or video(s) + input_videos = load_vision_input( + input_type=args.input_type, + batch_input_path=args.batch_input_path, + input_image_or_video_path=args.input_image_or_video_path, + data_resolution=args.data_resolution, + num_input_frames=args.num_input_frames, + ) + # Load input prompt(s) + if args.batch_input_path: + prompts_list = read_prompts_from_file(args.batch_input_path) + else: + prompts_list = [{"visual_input": args.input_image_or_video_path, "prompt": args.prompt}] + + # Iterate through prompts + for idx, prompt_entry in enumerate(prompts_list): + video_path = prompt_entry["visual_input"] + input_filename = os.path.basename(video_path) + + # Check if video exists in loaded videos + if input_filename not in input_videos: + log.critical(f"Input file {input_filename} not found, skipping prompt.") + continue + + inp_vid = input_videos[input_filename] + inp_prompt = prompt_entry["prompt"] + + # Generate video + log.info(f"Run with input: {prompt_entry}") + out_vid = pipeline.generate( + inp_prompt=inp_prompt, + inp_vid=inp_vid, + num_input_frames=args.num_input_frames, + seed=args.seed, + sampling_config=sampling_config, + ) + if out_vid is None: + log.critical("Guardrail blocked video2world generation.") + continue + + # Save video + if args.input_image_or_video_path: + out_vid_path = os.path.join(args.video_save_folder, f"{args.video_save_name}.mp4") + else: + out_vid_path = os.path.join(args.video_save_folder, f"{idx}.mp4") + imageio.mimsave(out_vid_path, out_vid, fps=25) + + log.info(f"Saved video to {out_vid_path}") + + +if __name__ == "__main__": + torch._C._jit_set_texpr_fuser_enabled(False) + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/world_generation_pipeline.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/world_generation_pipeline.py new file mode 100644 index 00000000..f6da7898 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/inference/world_generation_pipeline.py @@ -0,0 +1,908 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import os +from typing import List, Optional, Tuple + +import numpy as np +import torch +from einops import rearrange + +from cosmos1.models.autoregressive.configs.base.model_config import create_video2world_model_config +from cosmos1.models.autoregressive.configs.base.tokenizer import TokenizerConfig +from cosmos1.models.autoregressive.configs.inference.inference_config import ( + DataShapeConfig, + DiffusionDecoderSamplingConfig, + InferenceConfig, + SamplingConfig, +) +from cosmos1.models.autoregressive.diffusion_decoder.inference import diffusion_decoder_process_tokens +from cosmos1.models.autoregressive.diffusion_decoder.model import LatentDiffusionDecoderModel +from cosmos1.models.autoregressive.model import AutoRegressiveModel +from cosmos1.models.autoregressive.utils.inference import _SUPPORTED_CONTEXT_LEN, prepare_video_batch_for_saving +from cosmos1.models.common.base_world_generation_pipeline import BaseWorldGenerationPipeline +from cosmos1.models.diffusion.inference.inference_utils import ( + load_model_by_config, + load_network_model, + load_tokenizer_model, +) +from cosmos1.utils import log, misc + + +def detect_model_size_from_ckpt_path(ckpt_path: str) -> str: + """Detect model size from checkpoint path. + + Args: + ckpt_path: Path to model checkpoint file + + Returns: + str: Model size ('4b', '5b', '12b', or '13b') + + Examples: + >>> detect_model_size_from_ckpt_path("model_4B.pt") + '4b' + """ + model_size = "4b" + if "4B" in ckpt_path: + model_size = "4b" + elif "5B" in ckpt_path: + model_size = "5b" + elif "12B" in ckpt_path: + model_size = "12b" + elif "13B" in ckpt_path: + model_size = "13b" + else: + log.warning(f"Could not detect model size from checkpoint path: {ckpt_path}") + return model_size + + +def create_inference_config( + model_ckpt_path: str, + tokenizer_ckpt_path: str, + model_size: str = "4b", + batch_size: int = 1, + inference_type: str = "base", +) -> InferenceConfig: + """Create inference configuration for model. + + Args: + model_ckpt_path: Path to model checkpoint + tokenizer_ckpt_path: Path to tokenizer checkpoint + model_size: Size of model ('4b', '5b', '12b', '13b') + batch_size: Batch size for inference + inference_type: Type of inference ('base' or 'video2world') + + Returns: + InferenceConfig: Configuration object for inference + """ + model_size = model_size.lower() + # For inference config + kwargs = {} + if inference_type == "video2world": + kwargs.update( + dict( + insert_cross_attn=True, + insert_cross_attn_every_k_layers=1, + context_dim=1024, + training_type="text_to_video", + apply_abs_pos_emb=True, + ) + ) + if model_size == "5b": + model_size = "4b" # The base model (excluding the cross attention layers) is the 4B model + elif model_size == "13b": + model_size = "12b" # The base model (excluding the cross attention layers) is the 12B model + else: + raise ValueError(f"Unsupported model size for video2world inference_type: {model_size}") + else: + assert inference_type == "base", f"Unsupported inference_type: {inference_type}" + + model_config, tokenizer_config = create_video2world_model_config( + model_ckpt_path=model_ckpt_path, + tokenizer_ckpt_path=tokenizer_ckpt_path, + model_size=model_size, + rope_dim="3D", + add_special_tokens=False, + pixel_chunk_duration=33, + num_video_frames=33, + num_condition_latents_t=1, + batch_size=batch_size, + video_height=640, + video_width=1024, + **kwargs, + ) + + inference_config = InferenceConfig() + + inference_config.model_config = model_config + inference_config.tokenizer_config = tokenizer_config + + inference_config.data_shape_config = DataShapeConfig( + num_video_frames=model_config.num_video_frames, + height=model_config.video_height, + width=model_config.video_width, + latent_shape=model_config.video_latent_shape, + ) + inference_config.model_config.fuse_qkv = False + return inference_config + + +class ARBaseGenerationPipeline(BaseWorldGenerationPipeline): + """Base class for autoregressive world generation models. + + Handles the core functionality for generating videos using autoregressive models. + Provides configurable GPU memory management through model offloading and supports + different inference types for video generation. + + Attributes: + inference_config (InferenceConfig): Configuration for model inference + tokenizer_config (TokenizerConfig): Configuration for tokenizer + disable_diffusion_decoder (bool): Whether diffusion decoder is disabled + latent_shape (List[int]): Shape of video latents [T, H, W] + _supported_context_len (int): Supported context window length + latent_chunk_duration (int): Duration of latent chunks + pixel_chunk_duration (int): Duration of pixel chunks + diffusion_decoder_model (Optional[nn.Module]): The diffusion decoder model + """ + + def __init__( + self, + inference_type: str, + checkpoint_dir: str, + checkpoint_name: str, + has_text_input: bool = False, + offload_network: bool = False, + offload_tokenizer: bool = False, + disable_diffusion_decoder: bool = False, + offload_guardrail_models: bool = False, + offload_diffusion_decoder: bool = False, + ): + """Initialize the autoregressive world generation pipeline. + + Args: + inference_type: Type of world generation ('base' or 'video2world') + checkpoint_dir: Base directory containing model checkpoints + checkpoint_name: Name of the AR checkpoint to load + has_text_input: Whether the pipeline takes text input for world generation + disable_diffusion_decoder: Whether to disable the diffusion decoder stage + offload_network: Whether to offload AR model from GPU after use + offload_guardrail_models: Whether to offload content filtering models + offload_diffusion_decoder: Whether to offload diffusion decoder + + Raises: + AssertionError: If inference_type is not 'base' or 'video2world' + """ + assert inference_type in [ + "base", + "video2world", + ], "Invalid inference_type, must be 'base' or 'video2world'" + + # Create inference config + model_size = detect_model_size_from_ckpt_path(checkpoint_name) + model_ckpt_path = os.path.join(checkpoint_dir, checkpoint_name, "model.pt") + tokenizer_ckpt_path = os.path.join(checkpoint_dir, "Cosmos-1.0-Tokenizer-DV8x16x16/ema.jit") + + inference_config: InferenceConfig = create_inference_config( + model_ckpt_path=model_ckpt_path, + tokenizer_ckpt_path=tokenizer_ckpt_path, + model_size=model_size, + inference_type=inference_type, + ) + + self.inference_config = inference_config + self.disable_diffusion_decoder = disable_diffusion_decoder + + if not disable_diffusion_decoder: + self.diffusion_decoder_ckpt_path = os.path.join( + checkpoint_dir, "Cosmos-1.0-Diffusion-7B-Decoder-DV8x16x16ToCV8x8x8/model.pt" + ) + self.diffusion_decoder_config = "DD_FT_7Bv1_003_002_tokenizer888_spatch2_discrete_cond_on_token" + self.diffusion_decoder_tokenizer_path = os.path.join(checkpoint_dir, "Cosmos-1.0-Tokenizer-CV8x8x8") + self.dd_sampling_config = DiffusionDecoderSamplingConfig() + aux_vars_path = os.path.join(os.path.dirname(self.diffusion_decoder_ckpt_path), "aux_vars.pt") + # We use a generic prompt when no text prompts are available for diffusion decoder. + # Generic prompt used - "high quality, 4k, high definition, smooth video" + aux_vars = torch.load(aux_vars_path, weights_only=True) + self.generic_prompt = dict() + self.generic_prompt["context"] = aux_vars["context"].cuda() + self.generic_prompt["context_mask"] = aux_vars["context_mask"].cuda() + + self.latent_shape = inference_config.data_shape_config.latent_shape # [L, 40, 64] + self._supported_context_len = _SUPPORTED_CONTEXT_LEN + self.tokenizer_config = inference_config.tokenizer_config + + self.offload_diffusion_decoder = offload_diffusion_decoder + self.diffusion_decoder_model = None + if not self.offload_diffusion_decoder and not disable_diffusion_decoder: + self._load_diffusion_decoder() + + super().__init__( + inference_type=inference_type, + checkpoint_dir=checkpoint_dir, + checkpoint_name=checkpoint_name, + has_text_input=has_text_input, + offload_guardrail_models=offload_guardrail_models, + offload_network=offload_network, + offload_tokenizer=offload_tokenizer, + offload_text_encoder_model=True, + ) + + def _load_model(self): + """Load and initialize the autoregressive model. + + Creates and configures the autoregressive model with appropriate settings. + """ + self.model = AutoRegressiveModel( + config=self.inference_config.model_config, + ) + + def _load_network(self): + """Load network weights for the autoregressive model.""" + self.model.load_ar_model(tokenizer_config=self.inference_config.tokenizer_config) + + def _load_tokenizer(self): + """Load and initialize the tokenizer model. + + Configures the tokenizer using settings from inference_config and + attaches it to the autoregressive model. + """ + self.model.load_tokenizer(tokenizer_config=self.inference_config.tokenizer_config) + + def _load_diffusion_decoder(self): + """Load and initialize the diffusion decoder model.""" + self.diffusion_decoder_model = load_model_by_config( + config_job_name=self.diffusion_decoder_config, + config_file="cosmos1/models/autoregressive/diffusion_decoder/config/config_latent_diffusion_decoder.py", + model_class=LatentDiffusionDecoderModel, + ) + load_network_model(self.diffusion_decoder_model, self.diffusion_decoder_ckpt_path) + load_tokenizer_model(self.diffusion_decoder_model, self.diffusion_decoder_tokenizer_path) + + def _offload_diffusion_decoder(self): + """Offload diffusion decoder model from GPU memory.""" + if self.diffusion_decoder_model is not None: + del self.diffusion_decoder_model + self.diffusion_decoder_model = None + gc.collect() + torch.cuda.empty_cache() + + def _run_model_with_offload( + self, inp_vid: torch.Tensor, num_input_frames: int, seed: int, sampling_config: SamplingConfig + ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """Run the autoregressive model to generate video tokens. + + Takes input video frames and generates new video tokens using the autoregressive model. + Handles context frame selection and token generation. + + Args: + inp_vid (torch.Tensor): Input video tensor of shape + num_input_frames (int): Number of context frames to use from input. The tensor shape should be (B x T x 3 x H x W). + seed (int): Random seed for generation + sampling_config (SamplingConfig): Configuration for sampling parameters + + Returns: + tuple: ( + List of generated video tensors, + List of token index tensors, + List of prompt embedding tensors + ) + """ + # Choosing the context length from list of available contexts + latent_context_t_size = 0 + context_used = 0 + for _clen in self._supported_context_len: + if num_input_frames >= _clen: + context_used = _clen + latent_context_t_size += 1 + log.info(f"Using input size of {context_used} frames") + + data_batch = {"video": inp_vid} + data_batch = misc.to(data_batch, "cuda") + + T, H, W = self.latent_shape + num_gen_tokens = int(np.prod([T - latent_context_t_size, H, W])) + + out_videos_cur_batch, indices_tensor_cur_batch = self.generate_partial_tokens_from_data_batch( + data_batch=data_batch, + num_tokens_to_generate=num_gen_tokens, + sampling_config=sampling_config, + tokenizer_config=self.tokenizer_config, + latent_shape=self.latent_shape, + task_condition="video", + num_chunks_to_generate=1, + seed=seed, + ) + if self.offload_network: + self._offload_network() + if self.offload_tokenizer: + self._offload_tokenizer() + return out_videos_cur_batch, indices_tensor_cur_batch + + def _run_diffusion_decoder( + self, + out_videos_cur_batch: List[torch.Tensor], + indices_tensor_cur_batch: List[torch.Tensor], + t5_emb_batch: List[torch.Tensor], + ) -> List[torch.Tensor]: + """Process generated tokens through the diffusion decoder. + + Enhances video quality through diffusion-based decoding. + + Args: + out_videos_cur_batch: List of generated video tensors + indices_tensor_cur_batch: List of token indices tensors + t5_emb_batch: List of text embeddings for conditioning + + Returns: + list: Enhanced video tensors after diffusion processing + """ + out_videos_cur_batch_dd = diffusion_decoder_process_tokens( + model=self.diffusion_decoder_model, + indices_tensor=indices_tensor_cur_batch, + dd_sampling_config=self.dd_sampling_config, + original_video_example=out_videos_cur_batch[0], + t5_emb_batch=t5_emb_batch, + ) + return out_videos_cur_batch_dd + + def _run_diffusion_decoder_with_offload( + self, + out_videos_cur_batch: List[torch.Tensor], + indices_tensor_cur_batch: List[torch.Tensor], + t5_emb_batch: List[torch.Tensor], + ) -> List[torch.Tensor]: + """Run diffusion decoder with memory management. + + Loads decoder if needed, processes videos, and offloads decoder afterward + if configured in offload_diffusion_decoder. + + Args: + out_videos_cur_batch: List of generated video tensors + indices_tensor_cur_batch: List of token indices tensors + t5_emb_batch: List of text embeddings for conditioning + + Returns: + list: Enhanced video tensors after diffusion processing + """ + if self.offload_diffusion_decoder: + self._load_diffusion_decoder() + out_videos_cur_batch = self._run_diffusion_decoder( + out_videos_cur_batch, indices_tensor_cur_batch, t5_emb_batch + ) + if self.offload_diffusion_decoder: + self._offload_diffusion_decoder() + return out_videos_cur_batch + + def generate( + self, + inp_vid: torch.Tensor, + sampling_config: SamplingConfig, + num_input_frames: int = 9, + seed: int = 0, + ) -> np.ndarray | None: + """Generate a video continuation from input frames. + + Pipeline steps: + 1. Generates video tokens using autoregressive model + 2. Optionally enhances quality via diffusion decoder + 3. Applies safety checks if enabled + + Args: + inp_vid: Input video tensor of shape (batch_size, time, channels=3, height, width) + sampling_config: Parameters controlling the generation process + num_input_frames: Number of input frames to use as context (default: 9) + seed: Random seed for reproducibility (default: 0) + + Returns: + np.ndarray | None: Generated video as numpy array (time, height, width, channels) + if generation successful, None if safety checks fail + """ + log.info("Run generation") + out_videos_cur_batch, indices_tensor_cur_batch = self._run_model_with_offload( + inp_vid, num_input_frames, seed, sampling_config + ) + log.info("Finish AR model generation") + + if not self.disable_diffusion_decoder: + log.info("Run diffusion decoder on generated tokens") + out_videos_cur_batch = self._run_diffusion_decoder_with_offload( + out_videos_cur_batch, indices_tensor_cur_batch, t5_emb_batch=[self.generic_prompt["context"]] + ) + log.info("Finish diffusion decoder on generated tokens") + out_videos_cur_batch = prepare_video_batch_for_saving(out_videos_cur_batch) + output_video = out_videos_cur_batch[0] + + log.info("Run guardrail on generated video") + output_video = self._run_guardrail_on_video_with_offload(output_video) + if output_video is None: + log.critical("Generated video is not safe") + return None + log.info("Finish guardrail on generated video") + + return output_video + + @torch.inference_mode() + def generate_partial_tokens_from_data_batch( + self, + data_batch: dict, + num_tokens_to_generate: int, + sampling_config: SamplingConfig, + tokenizer_config: TokenizerConfig, + latent_shape: list[int], + task_condition: str, + num_chunks_to_generate: int = 1, + seed: int = 0, + ) -> tuple[List[torch.Tensor], List[torch.Tensor], List[torch.Tensor], List[torch.Tensor]]: + """Generate video tokens from partial input tokens with conditioning. + + Handles token generation and decoding process: + 1. Processes input batch and applies conditioning + 2. Generates specified number of new tokens + 3. Decodes tokens to video frames + + Args: + data_batch: Dictionary containing input data including video and optional context + num_tokens_to_generate: Number of tokens to generate + sampling_config: Configuration for sampling parameters + tokenizer_config: Configuration for tokenizer, including video tokenizer settings + latent_shape: Shape of video latents [T, H, W] + task_condition: Type of generation task ('video' or 'text_and_video') + num_chunks_to_generate: Number of chunks to generate (default: 1) + seed: Random seed for generation (default: 0) + + Returns: + tuple containing: + - List[torch.Tensor]: Generated videos + - List[torch.Tensor]: Input videos + - List[torch.Tensor]: Generated tokens + - List[torch.Tensor]: Token index tensors + """ + log.debug(f"Starting generate_partial_tokens_from_data_batch with seed {seed}") + log.debug(f"Number of tokens to generate: {num_tokens_to_generate}") + log.debug(f"Latent shape: {latent_shape}") + + video_token_start = tokenizer_config.video_tokenizer.tokenizer_offset + video_vocab_size = tokenizer_config.video_tokenizer.vocab_size + video_token_end = video_token_start + video_vocab_size + + logit_clipping_range = [video_token_start, video_token_end] + + if self.offload_network: + self._offload_network() + if self.offload_tokenizer: + self._load_tokenizer() + + assert logit_clipping_range == [ + 0, + self.model.tokenizer.video_vocab_size, + ], ( + f"logit_clipping_range {logit_clipping_range} is not supported for fast generate. Expected [0, {self.model.tokenizer.video_vocab_size}]" + ) + + out_videos = {} + out_indices_tensors = {} + + # for text2world, we only add a token at the beginning of the video tokens, this applies to 5B and 13B models + if self.model.tokenizer.tokenizer_config.training_type == "text_to_video": + num_bov_tokens = 1 + num_eov_tokens = 0 + else: + num_eov_tokens = 1 if self.model.tokenizer.tokenizer_config.add_special_tokens else 0 + num_bov_tokens = 1 if self.model.tokenizer.tokenizer_config.add_special_tokens else 0 + + chunk_idx = 0 + out_videos[chunk_idx] = [] + out_indices_tensors[chunk_idx] = [] + + # get the context embedding and mask + context = data_batch.get("context", None) if task_condition != "video" else None + context_mask = data_batch.get("context_mask", None) if task_condition != "video" else None + if context is not None: + context = misc.to(context, "cuda").detach().clone() + if context_mask is not None: + context_mask = misc.to(context_mask, "cuda").detach().clone() + + # get the video tokens + data_tokens, token_boundaries = self.model.tokenizer.tokenize(data_batch=data_batch) + data_tokens = misc.to(data_tokens, "cuda").detach().clone() + batch_size = data_tokens.shape[0] + + for sample_num in range(batch_size): + input_tokens = data_tokens[sample_num][0 : token_boundaries["video"][sample_num][1]] # [B, L] + input_tokens = [ + input_tokens[0 : -num_tokens_to_generate - num_eov_tokens].tolist() + ] # -1 is to exclude eov token + log.debug( + f"Run sampling. # input condition tokens: {len(input_tokens[0])}; # generate tokens: {num_tokens_to_generate + num_eov_tokens}; " + f"full length of the data tokens: {len(data_tokens[sample_num])}: {data_tokens[sample_num]}" + ) + video_start_boundary = token_boundaries["video"][sample_num][0] + num_bov_tokens + + video_decoded, indices_tensor = self.generate_video_from_tokens( + prompt_tokens=input_tokens, + latent_shape=latent_shape, + video_start_boundary=video_start_boundary, + max_gen_len=num_tokens_to_generate, + sampling_config=sampling_config, + logit_clipping_range=logit_clipping_range, + seed=seed, + context=context, + context_mask=context_mask, + ) # BCLHW, range [0, 1] + + # For the first chunk, we store the entire generated video + out_videos[chunk_idx].append(video_decoded[sample_num].detach().clone()) + out_indices_tensors[chunk_idx].append(indices_tensor[sample_num].detach().clone()) + + output_videos = [] + output_indice_tensors = [] + for sample_num in range(len(out_videos[0])): + tensors_to_concat = [out_videos[chunk_idx][sample_num] for chunk_idx in range(num_chunks_to_generate)] + concatenated = torch.cat(tensors_to_concat, dim=1) + output_videos.append(concatenated) + + indices_tensor_to_concat = [ + out_indices_tensors[chunk_idx][sample_num] for chunk_idx in range(num_chunks_to_generate) + ] + concatenated_indices_tensor = torch.cat(indices_tensor_to_concat, dim=1) # BLHW + output_indice_tensors.append(concatenated_indices_tensor) + + return output_videos, output_indice_tensors + + def generate_video_from_tokens( + self, + prompt_tokens: list[torch.Tensor], + latent_shape: list[int], + video_start_boundary: int, + max_gen_len: int, + sampling_config: SamplingConfig, + logit_clipping_range: list[int], + seed: int = 0, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + r""" + Function to generate video from input tokens. These input tokens can be initial text tokens (in case of text to video), + or partial ground truth tokens. + + Handles the core token-to-video generation process: + 1. Generates new tokens using the autoregressive model + 2. Handles padding and token sequence completion + 3. Reshapes and processes generated tokens + 4. Decodes final tokens into video frames + + Args: + model (AutoRegressiveModel): LLama model instance + prompt_tokens (list): Prompt tokens used by the model + latent_shape (list): Shape of the video latents + video_start_boundary (int): Index where the video tokens start + max_gen_len (int): Maximum length of the tokens that needs to be generated + sampling_config (SamplingConfig): Config used by sampler during inference + logit_clipping_range (list): Range of indices in the logits to be clipped, e.g. [video_token_start, video_token_end] + context (Optional[torch.Tensor]): The context tensor added via cross-attn. + context_mask (Optional[torch.Tensor]): The context cross-attn mask tensor. + Returns: + tuple containing: + - List[torch.Tensor]: Generated videos + - List[torch.Tensor]: Generated tokens + - List[torch.Tensor]: Token index tensors + """ + # Combine the tokens and do padding, sometimes the generated tokens end before the max_gen_len + total_seq_len = np.prod(latent_shape) + + assert not sampling_config.logprobs + + stop_tokens = self.model.tokenizer.stop_tokens + if self.offload_tokenizer: + self._offload_tokenizer() + if self.offload_network: + self._load_network() + + generation_tokens, _ = self.model.generate( + prompt_tokens=prompt_tokens, + temperature=sampling_config.temperature, + top_p=sampling_config.top_p, + echo=sampling_config.echo, + seed=seed, + context=context, + context_mask=context_mask, + max_gen_len=max_gen_len, + compile_sampling=sampling_config.compile_sampling, + compile_prefill=sampling_config.compile_prefill, + stop_tokens=stop_tokens, + verbose=True, + ) + generation_tokens = generation_tokens[:, video_start_boundary:] + # Combine the tokens and do padding, sometimes the generated tokens end before the max_gen_len + if generation_tokens.shape[1] < total_seq_len: + log.warning( + f"Generated video tokens (shape:{generation_tokens.shape}) shorted than expected {total_seq_len}. Could be the model produce end token early. Repeat the last token to fill the sequence in order for decoding." + ) + padding_len = total_seq_len - generation_tokens.shape[1] + padding_tokens = generation_tokens[:, [-1]].repeat(1, padding_len) + generation_tokens = torch.cat([generation_tokens, padding_tokens], dim=1) + # Cast to LongTensor + indices_tensor = generation_tokens.long() + # First, we reshape the generated tokens into batch x time x height x width + indices_tensor = rearrange( + indices_tensor, + "B (T H W) -> B T H W", + T=latent_shape[0], + H=latent_shape[1], + W=latent_shape[2], + ) + log.debug(f"generated video tokens {len(generation_tokens[0])} -> reshape: {indices_tensor.shape}") + # If logit clipping range is specified, offset the generated indices by the logit_clipping_range[0] + # Video decoder always takes tokens in the range (0, N-1). So, this offset is needed. + if len(logit_clipping_range) > 0: + indices_tensor = indices_tensor - logit_clipping_range[0] + + if self.offload_network: + self._offload_network() + if self.offload_tokenizer: + self._load_tokenizer() + + # Now decode the video using tokenizer. + video_decoded = self.model.tokenizer.video_tokenizer.decode(indices_tensor.cuda()) + # Normalize decoded video from [-1, 1] to [0, 1], and clip value + video_decoded = (video_decoded * 0.5 + 0.5).clamp_(0, 1) + return video_decoded, indices_tensor + + +class ARVideo2WorldGenerationPipeline(ARBaseGenerationPipeline): + """Video-to-world generation pipeline with text conditioning capabilities. + + Extends the base autoregressive generation pipeline by adding: + - Text prompt processing and embedding + - Text-conditioned video generation + - Additional safety checks for text input + - Memory management for text encoder model + + Enables generating video continuations that are guided by both + input video frames and text descriptions. + + Additional attributes compared to ARBaseGenerationPipeline: + offload_text_encoder_model (bool): Whether to offload text encoder from GPU after use + """ + + def __init__( + self, + checkpoint_dir: str, + checkpoint_name: str, + inference_type: str = None, + has_text_input: bool = True, + disable_diffusion_decoder: bool = False, + offload_guardrail_models: bool = False, + offload_diffusion_decoder: bool = False, + offload_network: bool = False, + offload_tokenizer: bool = False, + offload_text_encoder_model: bool = False, + ): + """Initialize text-conditioned video generation pipeline. + + Args: + checkpoint_dir: Base directory containing model checkpoints + checkpoint_name: Name of the checkpoint to load + inference_type: Type of world generation workflow + has_text_input: Whether the pipeline takes text input for world generation + disable_diffusion_decoder: Whether to disable diffusion decoder stage + offload_guardrail_models: Whether to offload content filtering models + offload_diffusion_decoder: Whether to offload diffusion decoder + offload_network: Whether to offload AR model from GPU + offload_tokenizer: Whether to offload tokenizer from GPU + offload_text_encoder_model: Whether to offload text encoder + """ + super().__init__( + checkpoint_dir=checkpoint_dir, + checkpoint_name=checkpoint_name, + inference_type=inference_type, + has_text_input=has_text_input, + disable_diffusion_decoder=disable_diffusion_decoder, + offload_guardrail_models=offload_guardrail_models, + offload_diffusion_decoder=offload_diffusion_decoder, + offload_network=offload_network, + offload_tokenizer=offload_tokenizer, + ) + self.offload_text_encoder_model = offload_text_encoder_model + if not self.offload_text_encoder_model: + self._load_text_encoder_model() + + def _run_model_with_offload( + self, + prompt_embedding: torch.Tensor, + prompt_mask: torch.Tensor, + inp_vid: torch.Tensor, + num_input_frames: int, + seed: int, + sampling_config: SamplingConfig, + ) -> tuple[List[torch.Tensor], List[torch.Tensor], List[torch.Tensor]]: + """Run model generation with memory management. + + Executes generation process and handles model offloading to manage GPU memory. + + Args: + prompt_embedding: Text prompt embeddings tensor + prompt_mask: Attention mask for prompt embeddings + inp_vid: Input video tensor + num_input_frames: Number of input frames to use + seed: Random seed for reproducibility + sampling_config: Configuration for sampling parameters + + Returns: + tuple: ( + List of generated video tensors + List of token index tensors + List of prompt embedding tensors + ) + """ + out_videos, indices_tensor, prompt_embedding = self._run_model( + prompt_embedding, prompt_mask, inp_vid, num_input_frames, seed, sampling_config + ) + if self.offload_network: + self._offload_network() + if self.offload_tokenizer: + self._offload_tokenizer() + return out_videos, indices_tensor, prompt_embedding + + def _run_model( + self, + prompt_embedding: torch.Tensor, + prompt_mask: torch.Tensor, + inp_vid: torch.Tensor, + num_input_frames: int, + seed: int, + sampling_config: SamplingConfig, + ) -> tuple[List[torch.Tensor], List[torch.Tensor], torch.Tensor]: + """Run core model generation process. + + Handles text-conditioned video generation: + 1. Prepares data batch with text embeddings and video + 2. Determines appropriate context length + 3. Generates video tokens with text conditioning + 4. Processes output tensors + + Args: + prompt_embedding: Text prompt embeddings tensor + prompt_mask: Attention mask for prompt embeddings + inp_vid: Input video tensor + num_input_frames: Number of input frames to use + seed: Random seed for reproducibility + sampling_config: Configuration for sampling parameters, + uses default config if None + + Returns: + tuple: ( + List of generated video tensors + List of token index tensors + Text context tensor + ) + """ + data_batch = {} + data_batch["context"], data_batch["context_mask"] = prompt_embedding, prompt_mask + T, H, W = self.latent_shape + + if sampling_config is None: + sampling_config = self.sampling_config + if type(inp_vid) is list: + batch_size = len(inp_vid) + elif type(inp_vid) is torch.Tensor: + batch_size = 1 + data_batch["context"] = data_batch["context"].repeat(batch_size, 1, 1) + data_batch["context_mask"] = data_batch["context_mask"].repeat(batch_size, 1) + data_batch["context_mask"] = torch.ones_like(data_batch["context_mask"]).bool() + + latent_context_t_size = 0 + + # Choosing the context length from list of available contexts + context_used = 0 + for _clen in self._supported_context_len: + if num_input_frames >= _clen: + context_used = _clen + latent_context_t_size += 1 + log.info(f"Using context of {context_used} frames") + + num_gen_tokens = int(np.prod([T - latent_context_t_size, H, W])) + + data_batch["video"] = inp_vid + data_batch["video"] = data_batch["video"].repeat(batch_size, 1, 1, 1, 1) + + data_batch = misc.to(data_batch, "cuda") + + log.debug(f" num_tokens_to_generate: {num_gen_tokens}") + log.debug(f" sampling_config: {sampling_config}") + log.debug(f" tokenizer_config: {self.tokenizer_config}") + log.debug(f" latent_shape: {self.latent_shape}") + log.debug(f" latent_context_t_size: {latent_context_t_size}") + log.debug(f" seed: {seed}") + + out_videos_cur_batch, indices_tensor_cur_batch = self.generate_partial_tokens_from_data_batch( + data_batch=data_batch, + num_tokens_to_generate=num_gen_tokens, + sampling_config=sampling_config, + tokenizer_config=self.tokenizer_config, + latent_shape=self.latent_shape, + task_condition="text_and_video", + seed=seed, + ) + return out_videos_cur_batch, indices_tensor_cur_batch, data_batch["context"] + + def generate( + self, + inp_prompt: str, + inp_vid: torch.Tensor, + num_input_frames: int = 9, + seed: int = 0, + sampling_config: SamplingConfig = None, + ) -> np.ndarray | None: + """Generate a video guided by text prompt and input frames. + + Pipeline steps: + 1. Validates text prompt safety if enabled + 2. Converts text to embeddings + 3. Generates video with text conditioning + 4. Enhances quality via diffusion decoder + 5. Applies video safety checks if enabled + + Args: + inp_prompt: Text prompt to guide the generation + inp_vid: Input video tensor with shape (batch_size, time, channels=3, height, width) + num_input_frames: Number of frames to use as context (default: 9) + seed: Random seed for reproducibility (default: 0) + sampling_config: Configuration for sampling parameters, + uses default config if None + + Returns: + np.ndarray | None: Generated video as numpy array (time, height, width, channels) + if generation successful, None if safety checks fail + """ + log.info("Run guardrail on prompt") + is_safe = self._run_guardrail_on_prompt_with_offload(inp_prompt) + if not is_safe: + log.critical("Input text prompt is not safe") + return None + log.info("Pass guardrail on prompt") + + log.info("Run text embedding on prompt") + prompt_embeddings, prompt_masks = self._run_text_embedding_on_prompt_with_offload([inp_prompt]) + prompt_embedding = prompt_embeddings[0] + prompt_mask = prompt_masks[0] + log.info("Finish text embedding on prompt") + + log.info("Run generation") + out_videos_cur_batch, indices_tensor_cur_batch, prompt_embedding = self._run_model_with_offload( + prompt_embedding, prompt_mask, inp_vid, num_input_frames, seed, sampling_config + ) + log.info("Finish AR model generation") + + if not self.disable_diffusion_decoder: + log.info("Run diffusion decoder on generated tokens") + out_videos_cur_batch = self._run_diffusion_decoder_with_offload( + out_videos_cur_batch, indices_tensor_cur_batch, [prompt_embedding] + ) + log.info("Finish diffusion decoder on generated tokens") + out_videos_cur_batch = prepare_video_batch_for_saving(out_videos_cur_batch) + output_video = out_videos_cur_batch[0] + + log.info("Run guardrail on generated video") + output_video = self._run_guardrail_on_video_with_offload(output_video) + if output_video is None: + log.critical("Generated video is not safe") + return None + log.info("Finish guardrail on generated video") + + return output_video diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/model.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/model.py new file mode 100644 index 00000000..dafd3208 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/model.py @@ -0,0 +1,600 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import json +import os +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +import torch +from safetensors.torch import load_file +from torch.nn.modules.module import _IncompatibleKeys + +from cosmos1.models.autoregressive.configs.base.model import ModelConfig +from cosmos1.models.autoregressive.configs.base.tokenizer import TokenizerConfig +from cosmos1.models.autoregressive.modules.mm_projector import MultimodalProjector +from cosmos1.models.autoregressive.networks.transformer import Transformer +from cosmos1.models.autoregressive.networks.vit import VisionTransformer, get_vit_config +from cosmos1.models.autoregressive.tokenizer.tokenizer import DiscreteMultimodalTokenizer, update_vocab_size +from cosmos1.models.autoregressive.utils.checkpoint import ( + get_partial_state_dict, + process_state_dict, + substrings_to_ignore, +) +from cosmos1.models.autoregressive.utils.sampling import decode_n_tokens, decode_one_token, prefill +from cosmos1.utils import log, misc + + +class AutoRegressiveModel(torch.nn.Module): + """ + A class to build and use a AutoRegressiveModel model for text generation. + + Methods: + build: Build a AutoRegressiveModel instance by initializing and loading a model checkpoint. + generate: Generate text sequences based on provided prompts using the language generation model. + """ + + def __init__( + self, + model: Transformer = None, + tokenizer: DiscreteMultimodalTokenizer = None, + config: ModelConfig = None, + vision_encoder: VisionTransformer = None, + mm_projector: MultimodalProjector = None, + ): + """ + Initialize the AutoRegressiveModel instance with a model and tokenizer. + + Args: + model (Transformer): The Transformer model for text generation. + tokenizer (Tokenizer): The tokenizer for encoding and decoding text. + config (Config): The configuration for the AutoRegressiveModel model. + vision_encoder (VisionTransformer): The vision encoder for the AutoRegressiveModel model. + mm_projector (MultimodalProjector): The multi-modal projector for the AutoRegressiveModel model. + """ + super().__init__() + self.model = model + self.tokenizer = tokenizer + self.config = config + + self.vision_encoder = vision_encoder + self.mm_projector = mm_projector + + @property + def precision(self): + return self.model.precision + + def get_num_params( + self, + ) -> int: + """ + Return the number of parameters in the model. + """ + n_params = sum(p.numel() for p in self.parameters()) + return n_params + + def load_ar_model( + self, + tokenizer_config, + ): + """ + Load the AR model. + """ + model_config = self.config + ckpt_path = model_config.ckpt_path + with misc.timer(f"loading checkpoint from {ckpt_path}"): + if ckpt_path.endswith("safetensors"): + # Load with safetensors API + checkpoint = load_file(ckpt_path, device="cpu") + else: + # The pytorch version + checkpoint = torch.load( + ckpt_path, + map_location="cpu", + mmap=True, # load the checkpoint in memory-mapped mode + weights_only=True, + ) + llm_checkpoint = checkpoint["model"] if "model" in checkpoint else checkpoint + orig_precision = torch.get_default_dtype() + precision = getattr(torch, model_config.precision) + torch.set_default_dtype(precision) + log.debug(f"Setting torch default dtype to {precision}") + + model = Transformer( + params=model_config, + tokenizer_config=tokenizer_config, + ) + log.debug( + f"tokenizer tokenizer_config.video_tokenizer.vocab_size {tokenizer_config.video_tokenizer.vocab_size}" + ) + vocab_size = update_vocab_size( + existing_vocab_size=0, + to_be_added_vocab_size=tokenizer_config.video_tokenizer.vocab_size, + training_type=tokenizer_config.training_type, + add_special_tokens=False, + ) + log.debug( + f"tokenizer tokenizer_config.video_tokenizer.vocab_size {tokenizer_config.video_tokenizer.vocab_size} vocab_size {vocab_size}" + ) + # Perform vocab expansion + if vocab_size > model.vocab_size: + log.debug(f"Expanding vocab size to {vocab_size}") + # For text-to-video training, we only expand the embedding layer but not the output (unembedding) layer, + expand_output_layer = not (tokenizer_config.training_type == "text_to_video") + model.expand_vocab( + vocab_size, + init_method="gaussian", + expand_output_layer=expand_output_layer, + ) + # Remove the "model." prefix in the state_dict + llm_checkpoint = process_state_dict(llm_checkpoint, prefix_to_remove="model.") + with misc.timer("loading state_dict into model"): + missing_keys, _ = model.load_state_dict(llm_checkpoint, strict=True) + # Remove keys with "_extra_state" suffix in missing_keys (defined by TransformerEngine for FP8 usage) + missing_keys = [k for k in missing_keys if not k.endswith("_extra_state")] + assert len(missing_keys) == 0, f"Missing keys: {missing_keys}" + + self.model = model.to(precision).to("cuda") + torch.set_default_dtype(orig_precision) # Reset the default dtype to the original value + + def load_tokenizer(self, tokenizer_config): + """ + Load the tokenizer. + """ + self.tokenizer = DiscreteMultimodalTokenizer(tokenizer_config) + + @staticmethod + def build( + model_config: ModelConfig = ModelConfig(), + tokenizer_config: TokenizerConfig = None, + ) -> "AutoRegressiveModel": + """ + Build a AutoRegressiveModel instance by initializing and loading a model checkpoint. + + Args: + model_config (ModelConfig, optional): The model configuration for the AutoRegressiveModel instance. Defaults to ModelConfig(). + tokenizer_config (TokenizerConfig, optional): The tokenizer configuration for the AutoRegressiveModel instance. Defaults to None. + download_rank_sync (bool, optional): Whether to download the checkpoint in a rank-synchronized manner. Defaults to True. + Returns: + AutoRegressiveModel: An instance of the AutoRegressiveModel class with the loaded model and tokenizer. + + Raises: + AssertionError: If there are no checkpoint files in the specified directory. + + Note: + This method sets the device to CUDA and loads the pre-trained model and tokenizer. + """ + # Initialize model configuration parameters + config_params = {} + + # Load checkpoint and model parameters + + if model_config.ckpt_path is None: + # If ckpt_path is not provided, we assume the model checkpoint is saved in the ckpt_dir + ckpt_dir = model_config.ckpt_dir + + # We prioritize safetensors version over the pytorch version, since the former is + # much faster for checkpoint loading. + checkpoints = sorted(Path(ckpt_dir).glob("*.safetensors")) + if len(checkpoints) == 0: + checkpoints = sorted(Path(ckpt_dir).glob("*.pth")) + + assert len(checkpoints) > 0, f"no checkpoint files found in {ckpt_dir}" + assert len(checkpoints) == 1, ( + f"multiple checkpoint files found in {ckpt_dir} (currently only one is supported)" + ) + ckpt_path = str(checkpoints[0]) # Assuming single checkpoint for non-parallel case + + if os.path.exists(Path(ckpt_dir) / "config.json"): + with open(Path(ckpt_dir) / "config.json", "r") as f: + config_params = json.loads(f.read()) + else: + log.info(f"No params.json found in the checkpoint directory ({ckpt_dir}). Using default model config.") + + else: + # If ckpt_path is provided, we load the model from the specified path, + # and use the default model configuration + ckpt_path = model_config.ckpt_path + + for key, value in config_params.items(): + if hasattr(model_config, key): + # Override the default model configuration with the parameters from the checkpoint + setattr(model_config, key, value) + + with misc.timer(f"loading checkpoint from {ckpt_path}"): + if ckpt_path.endswith("safetensors"): + # Load with safetensors API + checkpoint = load_file(ckpt_path, device="cpu") + else: + # The pytorch version + checkpoint = torch.load( + ckpt_path, + map_location="cpu", + mmap=True, # load the checkpoint in memory-mapped mode + weights_only=True, + ) + llm_checkpoint = checkpoint["model"] if "model" in checkpoint else checkpoint + + if model_config.vision_encoder is not None: + # Take the LLM weights (starting with "model.") from the VLM checkpoint + llm_checkpoint = get_partial_state_dict(llm_checkpoint, prefix="model.") + + vit_checkpoint = None + projector_checkpoint = None + if model_config.vision_encoder is not None: + # For vanilla VLM ckpt before fine-tuning, `checkpoint['model']` only contains LLM weights, and `checkpoint['vision_encoder']` + # and `checkpoint['mm_projector']` are both for those weights + # For fine-tuned VLM ckpt, `checkpoint['model']` contains all LLM, mm_projector and vision_encoder weights + if "vision_encoder" in checkpoint: + log.debug("Using pretrained vision_encoder") + vit_checkpoint = checkpoint["vision_encoder"] + else: + log.debug("Using fine-tuned vision_encoder") + vit_checkpoint = get_partial_state_dict(llm_checkpoint, prefix="vision_encoder.") + vit_checkpoint = process_state_dict(vit_checkpoint, prefix_to_remove="vision_encoder.") + if "mm_projector" in checkpoint: + log.debug("Using pretrained mm_projector") + projector_checkpoint = checkpoint["mm_projector"] + else: + log.debug("Using fine-tuned mm_projector") + projector_checkpoint = get_partial_state_dict(llm_checkpoint, prefix="mm_projector.") + projector_checkpoint = process_state_dict(projector_checkpoint, prefix_to_remove="mm_projector.") + assert len(vit_checkpoint) > 0 and len(projector_checkpoint) > 0, ( + "vit_checkpoint and projector_checkpoint cannot be empty. We do not support random initialization for vision_encoder and mm_projector." + ) + + tokenizer = DiscreteMultimodalTokenizer(tokenizer_config) + orig_precision = torch.get_default_dtype() + precision = getattr(torch, model_config.precision) + torch.set_default_dtype(precision) + log.debug(f"Setting torch default dtype to {precision}") + + model = Transformer( + params=model_config, + tokenizer_config=tokenizer_config, + ) + model_kwargs = {} + + if model_config.vision_encoder is not None: + assert model_config.mm_projector is not None, ( + "mm_projector must be provided if vision_encoder is provided." + ) + vit_config = get_vit_config(model_config.vision_encoder) + vision_encoder = VisionTransformer.build( + vit_config, + ) + + mm_projector = MultimodalProjector( + mm_projector_type=model_config.mm_projector, in_dim=vit_config["dim"], out_dim=model_config["dim"] + ) + model_kwargs.update({"vision_encoder": vision_encoder, "mm_projector": mm_projector}) + + # Perform vocab expansion + if tokenizer.vocab_size > model.vocab_size: + log.debug(f"Expanding vocab size to {tokenizer.vocab_size}") + # For text-to-video training, we only expand the embedding layer but not the output (unembedding) layer, + expand_output_layer = not (tokenizer.training_type == "text_to_video") + model.expand_vocab( + tokenizer.vocab_size, + init_method="gaussian", + expand_output_layer=expand_output_layer, + ) + + # Remove the "model." prefix in the state_dict + llm_checkpoint = process_state_dict(llm_checkpoint, prefix_to_remove="model.") + with misc.timer("loading state_dict into model"): + missing_keys, unexpected_keys = model.load_state_dict(llm_checkpoint, strict=True) + # Remove keys with "_extra_state" suffix in missing_keys (defined by TransformerEngine for FP8 usage) + missing_keys = [k for k in missing_keys if not k.endswith("_extra_state")] + assert len(missing_keys) == 0, f"Missing keys: {missing_keys}" + + if model_config.vision_encoder is not None: + vision_encoder.load_state_dict(vit_checkpoint) + mm_projector.load_state_dict(projector_checkpoint) + if model_config.vision_encoder_in_channels != 3: + vision_encoder.expand_in_channels(model_config.vision_encoder_in_channels) + + model = model.to(precision) # ensure model parameters are in the correct precision + log.debug(f"Model config: {model_config}") + + model_class = AutoRegressiveModel + + torch.set_default_dtype(orig_precision) # Reset the default dtype to the original value + + return model_class(model, tokenizer, model_config, **model_kwargs) + + @torch.no_grad() + def generate( + self, + prompt_tokens: List[List[int]] | torch.Tensor, + max_gen_len: int, + temperature: float = 1.0, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + num_gen_seq: int = 1, + logprobs: bool = False, + echo: bool = False, + seed: int = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + compile_sampling: bool = True, + compile_prefill: bool = False, + verbose: bool = True, + stop_tokens: Optional[Set[int]] = None, + images: Optional[torch.Tensor] = None, + ): + """ + Autoregressive generation built upon the gpt-fast implementation (https://github.com/pytorch-labs/gpt-fast). + + Args: + prompt_tokens (List[List[int]] | torch.Tensor): A single prompt of shape (1, seq_len). + max_gen_len (int): Maximum length of the generated text sequence. + temperature (float, optional): Temperature value for controlling randomness in sampling. Defaults to 0.6. + top_k (int, optional): Top-k value for top-k sampling. Defaults to None. + top_p (float, optional): Top-p probability threshold for nucleus sampling. Defaults to None. + num_gen_seq (int, optional): Number of outputs to generate given the same prompt. Defaults to 1. When temperature == 0, num_gen_seq must be 1 because the generation is deterministic. + echo (bool, optional): Flag indicating whether to include prompt tokens in the generated output. Defaults to False. + logit_clipping_range (list, optional): Range of logits to clip. Defaults to []. + seed (int, optional): Random seed for reproducibility. Defaults to None. + compile_sampling (bool, optional): Flag indicating whether to compile the decoding function. Defaults to True. + compile_prefill (bool, optional): Flag indicating whether to compile the prefill function. Defaults to False. + verbose (bool, optional): Flag indicating whether to print the the time. Defaults to False. + """ + assert top_k is None or top_p is None, f"Only one of top_k ({top_k} or top_p ({top_p} should be specified." + if temperature == 0: + top_p, top_k = None, None + log.debug("Setting top_p and top_k to None because temperature is 0") + if top_p is not None: + log.debug(f"Using top-p sampling with p={top_p} and temperature={temperature}") + elif top_k is not None: + log.debug(f"Using top-k sampling with k={top_k} and temperature={temperature}") + else: + log.debug("Not applying top-k or top-p sampling. Will use top-k sampling with k=None") + + orig_precision = torch.get_default_dtype() + torch.set_default_dtype(self.precision) + + torch._inductor.config.coordinate_descent_tuning = True + torch._inductor.config.triton.unique_kernel_names = True + # Experimental features to reduce compilation times, will be on by default in future + torch._inductor.config.fx_graph_cache = True + + if seed is not None: + misc.set_random_seed(seed) + + assert not logprobs, "logprobs are not supported for fast_generate yet" + # Examine if the function prefil and decode_one_token functions are compiled yet. If not, compile them based on the flags + if compile_sampling and not getattr(self, "inference_decode_compiled", False): + self.decode_one_token = torch.compile(decode_one_token, mode="reduce-overhead", fullgraph=True) + self.inference_decode_compiled = True + log.info("Compiled AR sampling function. Note: the first run will be slower due to compilation") + if compile_prefill and not getattr(self, "inference_prefill_compiled", False): + self.prefill = torch.compile(prefill, fullgraph=True, dynamic=True) + self.inference_prefill_compiled = True + log.info("Compiled prefill function. Note: the first run will be slower due to compilation") + + if not hasattr(self, "decode_one_token"): + self.decode_one_token = decode_one_token + if not hasattr(self, "prefill"): + self.prefill = prefill + + # Initialization and Assertions + if isinstance(self.model.params, list): + # During training, model.params is a list + log.debug( + f"Find self.model.params is a list, use self.config instead. Get max_batch_size={self.config.max_batch_size}, max_seq_len={self.config.max_seq_len}" + ) + params = self.config + else: + params = self.model.params + if isinstance(prompt_tokens, list): + prompt_tokens = torch.tensor(prompt_tokens, dtype=torch.long, device="cuda") + if prompt_tokens.ndim == 1: + prompt_tokens = prompt_tokens.view(1, -1) + else: + assert prompt_tokens.ndim == 2, f"prompt_tokens has shape {prompt_tokens.shape}" + batch_size, prompt_len = prompt_tokens.shape + total_len = min(params.max_seq_len, max_gen_len + prompt_len) + if max_gen_len + prompt_len > params.max_seq_len: + log.warning( + f"max_gen_len + prompt_len={max_gen_len + prompt_len} exceeds max_seq_len={params.max_seq_len}, truncate max_gen_len to {params.max_seq_len - prompt_len}" + ) + max_gen_len = params.max_seq_len - prompt_len + + if context_mask is not None: + context_mask = context_mask.to(dtype=torch.bool) + if context_mask.ndim == 2: + assert context_mask.shape[0] == batch_size, ( + f"batch_size mismatch: {context_mask.shape[0]} != {batch_size}" + ) + # Unsqueeze it to make it of shape [batch_size, 1, 1, context_seq_len] + context_mask = context_mask.view(batch_size, 1, 1, -1) + + if num_gen_seq > 1: + assert batch_size == 1, ( + f"num_gen_seq > 1 is only supported for a single prompt, got {len(prompt_tokens)} prompts" + ) + log.debug(f"Generating {num_gen_seq} sequences with the same prompt") + assert num_gen_seq <= params.max_batch_size, ( + f"num_gen_seq={num_gen_seq} exceeds max_batch_size={params.max_batch_size}" + ) + # repeat the prompt tokens for num_gen_seq times + prompt_tokens = prompt_tokens.repeat(num_gen_seq, 1) + assert prompt_tokens.shape == ( + num_gen_seq, + prompt_len, + ), f"prompt_tokens must be of shape (num_gen_seq, seq_len), got {prompt_tokens.shape}" + batch_size = len(prompt_tokens) + + # create an empty tensor of the expected final shape and fill in the current tokens + empty = torch.empty(batch_size, total_len, dtype=prompt_tokens.dtype, device=prompt_tokens.device) + empty[:, :prompt_len] = prompt_tokens + seq = empty + input_pos = torch.arange(0, prompt_len, device="cuda") + + if verbose: + prefill_start = time.time() + + if images is not None: + images = images.to(device=prompt_tokens.device, dtype=torch.bfloat16) + prompt_token_embeddings = self.embed_vision_language_features(prompt_tokens, images) + else: + prompt_token_embeddings = None + + if context is not None: + context = context.to(device=prompt_tokens.device, dtype=self.precision) + + # Prefill stage + next_token = self.prefill( + self.model, + input_pos=input_pos, + tokens=prompt_tokens if prompt_token_embeddings is None else None, + token_embeddings=prompt_token_embeddings, + temperature=temperature, + top_k=top_k, + top_p=top_p, + context=context, + context_mask=context_mask, + ) + if verbose: + prefill_time = time.time() - prefill_start + + seq[:, [prompt_len]] = next_token.to(dtype=seq.dtype) + input_pos = torch.tensor([prompt_len], dtype=torch.long, device="cuda") + stop_tokens = self.tokenizer.stop_tokens if stop_tokens is None else stop_tokens + stop_tokens = torch.tensor(list(stop_tokens), dtype=torch.long, device="cuda") + + if verbose: + decode_start = time.time() + # Decode stage + generated_tokens = decode_n_tokens( + self.model, + next_token.view(batch_size, -1), + input_pos, + max_gen_len - 1, + temperature=temperature, + top_k=top_k, + top_p=top_p, + stop_tokens=stop_tokens, + decode_one_token_function=self.decode_one_token, + context=context, + context_mask=context_mask, + ) + gen_len = len(generated_tokens) + if verbose: + decode_time = time.time() - decode_start + prefill_throughput = prompt_len / prefill_time + decode_throughput = gen_len / decode_time + log.debug(f"[Prefill] Time: {prefill_time:.2f}s; Throughput: {prefill_throughput:.2f} tokens/s") + log.debug(f"[Decode] Time: {decode_time:.2f}s; Throughput: {decode_throughput:.2f} tokens/s") + + generated_tokens = torch.cat(generated_tokens, dim=1) + + log.debug(f"generated_tokens: {generated_tokens.shape}") + seq = seq[:, : prompt_len + 1 + gen_len] + seq[:, prompt_len + 1 :] = generated_tokens + if not echo: + seq = seq[:, prompt_len:] + + torch.set_default_dtype(orig_precision) # Reset the default dtype to the original value + + return seq, None + + def embed_vision_language_features(self, input_ids: torch.Tensor, images: torch.tensor) -> torch.Tensor: + """ + Embed vision and language features into a combined representation. + + Args: + input_ids (torch.Tensor): Input token IDs. + images (torch.tensor): Input images. + + Returns: + torch.Tensor: Combined vision-language features. + + Raises: + AssertionError: If vision encoder or mm projector is not initialized, + or if dimensions mismatch. + """ + # Ensure vision encoder and mm projector are initialized + assert self.vision_encoder is not None + assert self.mm_projector is not None + + # Get image token ID and validate it + image_token_id = self.vision_encoder.image_token_id + assert isinstance(image_token_id, int) and image_token_id >= 0, f"Invalid image_token_id: {image_token_id}" + + # Identify text and image locations in the input + text_locations = input_ids != image_token_id + image_locations = input_ids == image_token_id + + # Process text features + text_features = self.model.tok_embeddings(input_ids[text_locations]) + + # Process image features + images = images.to(device=text_features.device, dtype=text_features.dtype) + vit_outputs = self.vision_encoder(images) + image_features = self.mm_projector(vit_outputs) + + # Get dimensions + B, seq_len = input_ids.shape + N_total = B * seq_len + N_txt, D_txt = text_features.shape + N_img, N_patch, D_img = image_features.shape + + # Reshape image features + image_features = image_features.reshape(N_img * N_patch, D_img) + + # Validate dimensions + assert D_txt == D_img, f"Text features dim {D_txt} should be equal to image features dim {D_img}" + assert N_total == N_txt + N_img * N_patch, ( + f"seq_len {seq_len} should be equal to N_txt + N_img*N_Patch {(N_txt, N_img * N_patch, image_locations.sum().item())}" + ) + + # Combine text and image features + combined_features = torch.empty( + (B, seq_len, D_txt), + dtype=text_features.dtype, + device=text_features.device, + ) + combined_features[text_locations, :] = text_features + combined_features[image_locations, :] = image_features + + return combined_features + + def state_dict(self, *args, **kwargs): + """ + Process the state dict (e.g., remove "_extra_state" keys imposed by TransformerEngine for FP8). + """ + state_dict = super().state_dict(*args, **kwargs) + return process_state_dict(state_dict) + + def load_state_dict(self, state_dict: Dict[str, Any], strict: bool = True, assign: bool = False): + """ + Ignore the missing keys with substrings matching `substring_to_ignore` (e.g., "_extra_state" keys imposed by + TransformerEngine for FP8). + """ + state_dict = process_state_dict(state_dict) + missing_keys, unexpected_keys = super().load_state_dict(state_dict, strict=False, assign=assign) + actual_missing_keys = [] + for key in missing_keys: + if not any(substring in key for substring in substrings_to_ignore): + actual_missing_keys.append(key) + if strict: + if len(actual_missing_keys) > 0 or len(unexpected_keys) > 0: + raise ValueError(f"Missing keys: {actual_missing_keys}\n\nUnexpected keys: {unexpected_keys}") + return _IncompatibleKeys(actual_missing_keys, unexpected_keys) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/attention.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/attention.py new file mode 100644 index 00000000..89066409 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/attention.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +from typing import Optional, Union + +import torch +from torch import nn + +from cosmos1.models.autoregressive.modules.embedding import RotaryPositionEmbedding +from cosmos1.models.autoregressive.modules.normalization import create_norm + + +class Attention(nn.Module): + """ + Attenion layer with KV cache. + """ + + def __init__( + self, + n_heads: int, + n_kv_heads: Union[int, None], + dim: int, + max_batch_size: int, + max_seq_len: int, + context_dim: Optional[int] = None, + use_qk_normalization: bool = False, + norm_type: str = "rmsnorm", + norm_eps: float = 1e-5, + causal_mask: Optional[bool] = True, + head_dim: Optional[int] = None, + fuse_qkv: bool = False, + precision: str = "bfloat16", + attn_type: str = "self", + ): + """ + Initializes the GQA module. + + Args: + n_heads (int): The number of attention heads. + n_kv_heads (int, optional): The number of key-value attention heads. None defaults to n_heads. + dim (int): The dimensionality of the input and output. + max_batch_size (int): The maximum batch size. + max_seq_len (int): The maximum sequence length. + context_dim (int, optional): The dimensionality of the context for cross-attn. Defaults to None. + use_qk_normalization (bool, optional): Whether to apply QK normalization. Defaults to False. + norm_type (str, optional): The type of normalization layer. Defaults to "rmsnorm". + norm_eps (float, optional): The epsilon value for normalization. Defaults to 1e-5. + causal_mask (bool, optional): Whether to use causal mask. Defaults to True. + head_dim (int, optional): The dimensionality of each attention head. If None, defaults to dim // n_heads. + fuse_qkv (bool, optional): Whether to fuse QKV. Defaults to False. + precision (str, optional): The precision of the module. Defaults to "bfloat16". + attn_type (str, optional): The type of attention. Defaults to "self". + """ + super().__init__() + assert attn_type in ["self", "cross", "full"], f"Invalid attention type: {attn_type}" + self.attn_type = attn_type + context_dim = dim if context_dim is None else context_dim + + self.dim = dim + self.context_dim = context_dim + self.n_kv_heads = n_heads if n_kv_heads is None else n_kv_heads + self.n_local_kv_heads = self.n_kv_heads + self.n_local_heads = n_heads + self.n_rep = self.n_local_heads // self.n_local_kv_heads + self.head_dim = dim // n_heads if head_dim is None else head_dim + self.causal_mask = causal_mask + self.fuse_qkv = fuse_qkv + self.precision = precision + + if fuse_qkv: + assert context_dim == dim, f"Fuse QKV requires context_dim ({context_dim}) to be equal to dim ({dim})" + self.total_local_head_dim = (self.n_local_heads + 2 * self.n_local_kv_heads) * self.head_dim + self.wqkv = nn.Linear(dim, self.total_local_head_dim, bias=False) + # Register hook to load fused QKV weights + self._register_load_state_dict_pre_hook(self.load_hook) + else: + self.wq = nn.Linear(dim, self.n_local_heads * self.head_dim, bias=False) + self.wk = nn.Linear(context_dim, self.n_local_kv_heads * self.head_dim, bias=False) + self.wv = nn.Linear(context_dim, self.n_local_kv_heads * self.head_dim, bias=False) + self.wo = nn.Linear(self.n_local_heads * self.head_dim, dim, bias=False) + + self.max_batch_size = max_batch_size + self.max_seq_len = max_seq_len + + if self.attn_type == "self": + # Cache for key and value tensors + self.init_kv_cache() + + # QK normalization layers + if use_qk_normalization: + self.q_norm = create_norm(norm_type, dim=self.head_dim, eps=norm_eps) + self.k_norm = create_norm(norm_type, dim=self.head_dim, eps=norm_eps) + + self.use_qk_normalization = use_qk_normalization + + self.to(dtype=getattr(torch, self.precision)) + + def load_hook(self, state_dict, prefix, *args): + if prefix + "wq.weight" in state_dict: + wq = state_dict.pop(prefix + "wq.weight") + wk = state_dict.pop(prefix + "wk.weight") + wv = state_dict.pop(prefix + "wv.weight") + state_dict[prefix + "wqkv.weight"] = torch.cat([wq, wk, wv]) + + def init_kv_cache(self, dtype=None): + cache_shape = (self.max_batch_size, self.n_local_kv_heads, self.max_seq_len, self.head_dim) + if dtype is None: + dtype = getattr(torch, self.precision) + if self.attn_type == "self": + self.cache_k = torch.zeros(cache_shape, dtype=dtype).cuda() + self.cache_v = torch.zeros(cache_shape, dtype=dtype).cuda() + + def forward( + self, + x: torch.Tensor, + rope: RotaryPositionEmbedding, + input_pos: torch.Tensor, + mask: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + ): + """ + Forward pass of GQA. + + Args: + x: The input tensor of shape (batch_size, seq_len, dim). + rope: The rotary positional embedding module. + input_pos: The starting position of the current sequence. + mask: The attention mask tensor. + context: The context tensor of shape (batch_size, context_len, dim). + + Returns: + The output tensor after applying GQA. + """ + bsz, seqlen, _ = x.shape + + # Use one single module to handle both self-attn and cross-attn + context = x if context is None else context + context_len = seqlen if context is None else context.shape[1] + + if self.fuse_qkv: + q_size = self.n_local_heads * self.head_dim + kv_size = self.n_local_kv_heads * self.head_dim + xq, xk, xv = self.wqkv(x).split([q_size, kv_size, kv_size], dim=-1) + else: + # Compute query, key, and value projections + xq, xk, xv = self.wq(x), self.wk(context), self.wv(context) + + # Reshape projections + xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim) + xk = xk.view(bsz, context_len, self.n_local_kv_heads, self.head_dim) + xv = xv.view(bsz, context_len, self.n_local_kv_heads, self.head_dim) + + # QK normalization + if self.use_qk_normalization: + xq = self.q_norm(xq) + xk = self.k_norm(xk) + + # Apply rotary positional embeddings to queries and keys + # Only apply RoPE to self-attention! + if self.attn_type in ["self", "full"]: + xq, xk = rope(xq, xk, input_pos, seqlen) + + xq, xk, xv = map(lambda x: x.transpose(1, 2), (xq, xk, xv)) + # xq: (bs, n_local_heads, seqlen, head_dim) + # xk: (bs, n_kv_heads, cache_len + context_len, head_dim) + # xv: (bs, n_kv_heads, cache_len + context_len, head_dim) + if self.attn_type == "self": + # Update cache with current key and value tensors + assert input_pos is not None + self.cache_k[:bsz, :, input_pos] = xk + self.cache_v[:bsz, :, input_pos] = xv + keys, values = ( + self.cache_k[:bsz, :, :], + self.cache_v[:bsz, :, :], + ) + else: + keys, values = xk, xv + + # Repeat keys and values if necessary + keys = keys.repeat_interleave(self.n_rep, dim=1) # (bs, n_local_heads, cache_len + context_len, head_dim) + values = values.repeat_interleave(self.n_rep, dim=1) # (bs, n_local_heads, cache_len + context_len, head_dim) + + # For self-attention, `is_causal` should be set to False when KV cache is pre-computed and used, + # since the masking is handled outside this attention module. + # For cross-attention, it's always full-attn without causal mask + is_causal = False + output = scaled_dot_product_attention( + xq, + keys, + values, + head_dim=self.head_dim, + mask=mask, + is_causal=is_causal, + dropout_p=0.0, + ) + output = output.view(bsz, seqlen, -1) + output = self.wo(output) + return output + + +def scaled_dot_product_attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + head_dim: int, + mask: Optional[torch.Tensor] = None, + is_causal: Optional[bool] = None, + dropout_p: float = 0.0, +) -> torch.Tensor: + """ + PyTorch's native implementation of Flash Attention 2. + + If `is_causal` is given, then the causal attention mask is applied accordingly: + - If `is_causal` is True, the standard upper-left causal attention masking is applied. + - If `is_causal` is False, no attention mask is applied, unless an explicit mask tensor is + provided (i.e., `mask is not None`). + + If `is_causal` is not given (i.e., `is_causal is None`), then the attention mask is applied + based on the provided mask tensor: + - If no explicit attention mask is given (i.e., `mask is None`), `is_causal` is set to True, + leading to the standard upper-left causal attention masking. + - If an attention mask is given (i.e., `mask is not None`), the provided mask is used, + and `is_causal` is set to False. + + Args: + q (torch.Tensor): Query tensor + k (torch.Tensor): Key tensor + v (torch.Tensor): Value tensor + head_dim (int): Dimension of each attention head + mask (Optional[torch.Tensor], optional): Attention mask. Defaults to None. + is_causal (Optional[bool], optional): Whether to apply causal attention mask. Defaults to None. + dropout_p (float, optional): Dropout rate. Defaults to 0.0. + + Returns: + torch.Tensor: Output tensor after applying scaled dot-product attention + """ + scale = 1.0 / math.sqrt(head_dim) + if is_causal is None: + is_causal = mask is None + y = torch.nn.functional.scaled_dot_product_attention( + q, + k, + v, + attn_mask=mask, + dropout_p=dropout_p, + scale=scale, + is_causal=is_causal, + ) + return y.transpose(1, 2).contiguous() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/embedding.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/embedding.py new file mode 100644 index 00000000..c59bbbab --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/embedding.py @@ -0,0 +1,495 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +from typing import List, Optional, Tuple + +import numpy as np +import torch +from einops import rearrange, repeat + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +def _rotate_half_te(x: torch.Tensor) -> torch.Tensor: + """ + change sign so the last dimension becomes [-odd, +even]. + Adopted from TransformerEngine. + Source: https://github.com/NVIDIA/TransformerEngine/blob/main/transformer_engine/pytorch/attention.py + """ + x = x.view(x.shape[:-1] + torch.Size((2, x.shape[-1] // 2))) + x1, x2 = x.unbind(dim=-2) + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rotary_pos_emb_te( + t: torch.Tensor, + cos_freqs: torch.Tensor, + sin_freqs: torch.Tensor, +) -> torch.Tensor: + """ + Apply rotary positional embedding tensor to the input tensor. + Adopted from TransformerEngine. + Source: https://github.com/NVIDIA/TransformerEngine/blob/main/transformer_engine/pytorch/attention.py + + Parameters + ---------- + t: torch.Tensor + Input tensor of shape `[b, s, h, d]`, on which + rotary positional embedding will be applied. + cos_freqs: torch.Tensor + Cosine component of rotary positional embedding tensor of shape `[s, 1, 1, d]` and dtype 'float', + sin_freqs: torch.Tensor + Sine component of rotary positional embedding tensor of shape `[s, 1, 1, d]` and dtype 'float', + """ + rot_dim = cos_freqs.shape[-1] + # ideally t_pass is empty so rotary pos embedding is applied to all tensor t + t, t_pass = t[..., :rot_dim], t[..., rot_dim:] + # first part is cosine component + # second part is sine component, need to change signs with _rotate_half method + t = (t * cos_freqs) + (_rotate_half_te(t) * sin_freqs) + output = torch.cat((t, t_pass), dim=-1) + return output + + +class RotaryPositionEmbedding(torch.nn.Module): + """ + Rotary Position Embedding module as described in the paper: + https://arxiv.org/abs/2104.09864 + + This module implements rotary positional embeddings, which are used to + enhance the performance of transformer models. + + Args: + dim (int): Dimensionality of the input tensor. + max_position_embeddings (Optional[int]): Maximum position embeddings. + original_max_position_embeddings (Optional[int]): Original maximum position embeddings. + rope_theta (Optional[float]): Base for the frequency calculation. + apply_yarn (Optional[bool]): Whether to apply YaRN (Yet another Rotary). + scale (Optional[int]): Scaling factor for the frequency calculation. + extrapolation_factor (Optional[int]): Extrapolation factor for the frequency extension. + attn_factor (Optional[int]): Attention factor for the frequency calculation. + beta_fast (Optional[int]): Fast beta value for the YaRN frequency calculation. + beta_slow (Optional[int]): Slow beta value for the YaRN frequency calculation. + rope_dim (Optional[str]): Dimensionality of the RoPE. Choices: "1D", "2D", "3D". + latent_shape (Optional[List[int]]): Shape of the latent tensor for video or image inputs. + original_latent_shape (Optional[List[int]]): Original shape of the latent tensor for video or image inputs. + pad_to_multiple_of (Optional[int]): Pad the position embedding to a multiple of this value. + """ + + def __init__( + self, + dim: int, + max_position_embeddings: Optional[int] = None, + original_max_position_embeddings: Optional[int] = None, + rope_theta: Optional[float] = 10000.0, + apply_yarn: Optional[bool] = False, + scale: Optional[int] = None, + extrapolation_factor: Optional[int] = 1, + attn_factor: Optional[int] = 1, + beta_fast: Optional[int] = 32, + beta_slow: Optional[int] = 1, + rope_dim: Optional[str] = "1D", + latent_shape: Optional[List[int]] = None, + original_latent_shape: Optional[List[int]] = None, + pad_to_multiple_of: Optional[int] = None, + ): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.rope_theta = rope_theta + self.apply_yarn = apply_yarn + self.scale = scale + self.extrapolation_factor = extrapolation_factor + self.attn_factor = attn_factor + self.beta_fast = beta_fast + self.beta_slow = beta_slow + self.mscale = 1.0 + self.rope_dim = rope_dim + self.latent_shape = latent_shape + self.original_latent_shape = original_latent_shape + self.pad_to_multiple_of = pad_to_multiple_of + self.get_inv_freq(torch.cuda.current_device()) + + def get_mscale(self, scale: float = 1.0) -> float: + """Get the magnitude scaling factor for YaRN.""" + if scale <= 1: + return 1.0 + return 0.1 * math.log(scale) + 1.0 + + def forward(self, seq_len: Optional[int] = None) -> torch.Tensor: + """ + Forward pass for the rotary position embedding. + + Args: + seq_len (Optional[int]): Length of the sequence. + + Returns: + torch.Tensor: The computed frequencies for positional embedding. + """ + + if self.apply_yarn and seq_len > self.max_seq_len_cached: + self.max_seq_len_cached = seq_len + self.freqs = self.compute_freqs() + + return self.freqs + + def compute_freqs( + self, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute the spatial frequencies for the latent tensor.""" + self.seq = torch.arange(self.max_seq_len_cached, dtype=torch.float).cuda() + if self.rope_dim == "1D": + emb = torch.einsum("i,j->ij", self.seq, self.inv_freq) + + elif self.rope_dim == "2D": + H, W = self.latent_shape + half_emb_h = torch.outer(self.seq[:H], self.spatial_inv_freq) + half_emb_w = torch.outer(self.seq[:W], self.spatial_inv_freq) + emb = torch.cat( + [ + repeat(half_emb_h, "h d -> h w d", w=W), + repeat(half_emb_w, "w d -> h w d", h=H), + ] + * 2, + dim=-1, + ) + emb = rearrange(emb, "h w d -> (h w) 1 1 d").float() + + elif self.rope_dim == "3D": + T, H, W = self.latent_shape + half_emb_t = torch.outer(self.seq[:T], self.temporal_inv_freq) + half_emb_h = torch.outer(self.seq[:H], self.spatial_inv_freq) + half_emb_w = torch.outer(self.seq[:W], self.spatial_inv_freq) + emb = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + emb = rearrange(emb, "t h w d -> (t h w) 1 1 d").float() + else: + raise ValueError(f"Invalid RoPE dimensionality: {self.rope_dim}") + return emb + + def get_scale_factors(self, inv_freq: torch.Tensor, original_seq_len: int) -> torch.Tensor: + """Get the scale factors for YaRN.""" + # Calculate the high and low frequency cutoffs for YaRN. Note: `beta_fast` and `beta_slow` are called + # `high_freq_factor` and `low_freq_factor` in the Llama 3.1 RoPE scaling code. + high_freq_cutoff = 2 * math.pi * self.beta_fast / original_seq_len + low_freq_cutoff = 2 * math.pi * self.beta_slow / original_seq_len + # Obtain a smooth mask that has a value of 0 for low frequencies and 1 for high frequencies, with linear + # interpolation in between. + smooth_mask = torch.clamp((inv_freq - low_freq_cutoff) / (high_freq_cutoff - low_freq_cutoff), min=0, max=1) + # For low frequencies, we scale the frequency by 1/self.scale. For high frequencies, we keep the frequency. + scale_factors = (1 - smooth_mask) / self.scale + smooth_mask + return scale_factors + + def get_inv_freq(self, device: torch.device) -> None: + """Get the inverse frequency.""" + if self.rope_dim == "1D": + assert self.max_position_embeddings is not None, "Max position embeddings required." + inv_freq = 1.0 / ( + self.rope_theta ** (torch.arange(0, self.dim, 2, dtype=torch.float32, device=device) / self.dim) + ) + if self.apply_yarn: + assert self.original_max_position_embeddings is not None, "Original max position embeddings required." + assert self.beta_slow is not None, "Beta slow value required." + assert self.beta_fast is not None, "Beta fast value required." + + scale_factors = self.get_scale_factors(inv_freq, self.original_max_position_embeddings) + # Apply the scaling factors to inv_freq. + inv_freq = inv_freq * scale_factors + # Set the magnitude scaling factor. + self.mscale = float(self.get_mscale(self.scale) * self.attn_factor) + self.max_seq_len_cached = self.max_position_embeddings + self.inv_freq = inv_freq + + elif self.rope_dim == "2D": + assert self.latent_shape is not None, "Latent shape required." + dim_h = self.dim // 2 + spatial_inv_freq = 1.0 / ( + self.rope_theta ** torch.arange(0, dim_h, 2, dtype=torch.float32, device=device) / dim_h + ) + if self.apply_yarn: + assert self.original_latent_shape is not None, "Original latent shape required." + assert self.beta_slow is not None, "Beta slow value required." + assert self.beta_fast is not None, "Beta fast value required." + + scale_factors = self.get_scale_factors(spatial_inv_freq, self.original_latent_shape[0]) + spatial_inv_freq = spatial_inv_freq * scale_factors + self.mscale = float(self.get_mscale(self.scale) * self.attn_factor) + self.spatial_inv_freq = spatial_inv_freq + self.max_seq_len_cached = max(self.latent_shape) + + elif self.rope_dim == "3D": + assert self.latent_shape is not None, "Latent shape required." + dim_h = self.dim // 6 * 2 + dim_t = self.dim - 2 * dim_h + self.dim_spatial_range = torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().to(device) / dim_h + spatial_inv_freq = 1.0 / (self.rope_theta**self.dim_spatial_range) + self.dim_temporal_range = torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().to(device) / dim_t + temporal_inv_freq = 1.0 / (self.rope_theta**self.dim_temporal_range) + if self.apply_yarn: + assert self.original_latent_shape is not None, "Original latent shape required." + assert self.beta_slow is not None, "Beta slow value required." + assert self.beta_fast is not None, "Beta fast value required." + scale_factors_spatial = self.get_scale_factors(spatial_inv_freq, self.original_latent_shape[1]) + spatial_inv_freq = spatial_inv_freq * scale_factors_spatial + scale_factors_temporal = self.get_scale_factors(temporal_inv_freq, self.original_latent_shape[0]) + temporal_inv_freq = temporal_inv_freq * scale_factors_temporal + self.mscale = float(self.get_mscale(self.scale) * self.attn_factor) + self.spatial_inv_freq = spatial_inv_freq + self.temporal_inv_freq = temporal_inv_freq + self.max_seq_len_cached = max(self.latent_shape) + else: + raise ValueError(f"Invalid RoPE dimensionality: {self.rope_dim}") + + self.freqs = self.compute_freqs() + + +class RotaryPositionEmbeddingPytorchV2(RotaryPositionEmbedding): + """ + Rotary Position Embedding that works in the same way as the TransformerEngine RoPE + (https://github.com/NVIDIA/TransformerEngine/blob/main/transformer_engine/pytorch/attention.py) + + """ + + def __init__( + self, + seq_len: int, + training_type: str = None, + **kwargs, + ): + super().__init__( + **kwargs, + ) + emb = self.create_rope_freqs(seq_len=seq_len, training_type=training_type) + emb = emb.transpose(0, 1).contiguous() # [seq, 1, 1, dim] -> [1, seq, 1, dim] + assert emb.shape[0] == 1 and emb.shape[2] == 1, f"emb shape: {emb.shape}" + # cos/sin first then dtype conversion for better precision + self.register_buffer("cos_cached", torch.cos(emb), persistent=False) + self.register_buffer("sin_cached", torch.sin(emb), persistent=False) + + def create_rope_freqs(self, seq_len: int, training_type: str = None) -> torch.Tensor: + """ + Create rotary position embedding frequencies. + + Args: + seq_len (int): Sequence length of a sample. + + Returns: + torch.Tensor: The computed positional embeddings. + """ + if self.rope_dim == "1D": + freqs = super().forward(seq_len=seq_len) + emb = torch.cat((freqs, freqs), dim=-1) + emb = emb.reshape(emb.size(0), 1, 1, emb.size(1)) + + elif self.rope_dim in ["2D", "3D"]: + emb = super().forward(seq_len=seq_len) + if training_type == "text_to_video": + # since we added token at the beginning of the video for text2world, we also extend the position embedding by one token in the beginning + bov_pe = torch.zeros((1, *emb.shape[1:]), device=emb.device) + emb = torch.cat((bov_pe, emb), dim=0) + else: + raise ValueError(f"Invalid RoPE dimensionality: {self.rope_dim}") + if self.pad_to_multiple_of is not None and emb.shape[0] % self.pad_to_multiple_of != 0: + # Round up to the nearest multiple of pad_to_multiple_of + pad_len = self.pad_to_multiple_of - emb.shape[0] % self.pad_to_multiple_of + emb = torch.cat((emb, torch.zeros((pad_len, *emb.shape[1:]), device=emb.device)), dim=0) + + return emb + + def forward( + self, q: torch.Tensor, k: torch.Tensor, input_pos: Optional[torch.Tensor] = None, seq_len: Optional[int] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + if q.dtype != self.cos_cached.dtype: + self.cos_cached = self.cos_cached.to(q.dtype) + self.sin_cached = self.sin_cached.to(q.dtype) + + cos_emb = self.cos_cached + sin_emb = self.sin_cached + if input_pos is not None: + cos_emb = cos_emb[:, input_pos, :, :] + sin_emb = sin_emb[:, input_pos, :, :] + elif seq_len is not None: + cos_emb = cos_emb[:, :seq_len, :, :] + sin_emb = sin_emb[:, :seq_len, :, :] + q = _apply_rotary_pos_emb_te(q, cos_emb, sin_emb) + k = _apply_rotary_pos_emb_te(k, cos_emb, sin_emb) + return q, k + + +class RotaryPositionEmbeddingPytorchV1(RotaryPositionEmbedding): + """ + Rotary Position Embedding that works in the same way as + mistral_inference (https://github.com/mistralai/mistral-inference/blob/main/src/mistral_inference/rope.py) + or llama3 (https://github.com/meta-llama/llama3/blob/main/llama/model.py) + + """ + + def __init__( + self, + **kwargs, + ): + super().__init__( + **kwargs, + ) + emb = None + if self.rope_dim == "1D": + emb = torch.stack((self.freqs, self.freqs), dim=-1).reshape(*self.freqs.shape[:-1], -1) + elif self.rope_dim in ["2D", "3D"]: + emb = rearrange(self.freqs, "s 1 1 d -> s d").float() + self.register_buffer("cos_cached", (emb.cos() * self.mscale)[None, :, None, :], persistent=False) + self.register_buffer("sin_cached", (emb.sin() * self.mscale)[None, :, None, :], persistent=False) + + def rotate_half(self, x: torch.Tensor) -> torch.Tensor: + """Rotate half the hidden dimensions of the input tensor.""" + x_reshaped = x.reshape(*x.shape[:-1], -1, 2) + x1 = x_reshaped[..., 0] + x2 = x_reshaped[..., 1] + output = torch.stack((-x2, x1), dim=-1).reshape(*x.shape) + return output + + def forward( + self, q: torch.Tensor, k: torch.Tensor, input_pos: Optional[torch.Tensor] = None, seq_len: Optional[int] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Forward pass for the rotary position embedding. + + Args: + q (torch.Tensor): Query tensor. + k (torch.Tensor): Key tensor. + input_pos (Optional[torch.Tensor]): Starting position for the sequence. + seq_len (Optional[int]): Length of the sequence. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Rotated query and key tensors. + """ + if self.apply_yarn and seq_len > self.max_seq_len_cached: + freqs = super().forward(seq_len) + if self.rope_dim == "1D": + emb = torch.stack((freqs, freqs), dim=-1).reshape(*freqs.shape[:-1], -1) + elif self.rope_dim in ["2D", "3D"]: + emb = rearrange(freqs, "s 1 1 d -> s d").float() + else: + raise ValueError(f"Invalid RoPE dimensionality: {self.rope_dim}") + self.register_buffer( + "cos_cached", (emb.cos() * self.mscale)[None, :, None, :].to(q.dtype), persistent=False + ) + self.register_buffer( + "sin_cached", (emb.sin() * self.mscale)[None, :, None, :].to(q.dtype), persistent=False + ) + + if input_pos is not None: + cos_cached = self.cos_cached[:, input_pos] + sin_cached = self.sin_cached[:, input_pos] + else: + assert self.cos_cached.shape[1] >= seq_len, ( + f"Invalid sequence length; cos_cached.shape {self.cos_cached.shape}, seq_len {seq_len}." + ) + cos_cached = self.cos_cached[:, :seq_len, ...] + sin_cached = self.sin_cached[:, :seq_len, ...] + xq = q * cos_cached + self.rotate_half(q) * sin_cached + xk = k * cos_cached + self.rotate_half(k) * sin_cached + + return xq.type_as(q), xk.type_as(k) + + +class SinCosPosEmbAxisTE(torch.nn.Module): + def __init__( + self, + dim: int, + latent_shape: Optional[List[int]] = None, + pad_to_multiple_of: Optional[int] = None, + dtype: torch.dtype = torch.bfloat16, + device="cuda", + **kwargs, + ): + """ + Args: + dim (int): Dimensionality of the input tensor. + latent_shape (Optional[List[int]]): Shape of the latent tensor for video or image inputs. + pad_to_multiple_of (Optional[int]): Pad the position embedding to a multiple of this value. + dtype (torch.dtype): Data type of the position embedding tensor. + """ + super().__init__() + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + self.latent_shape = latent_shape + T, H, W = latent_shape + emb_h = get_1d_sincos_pos_embed_from_grid(dim_h, pos=np.arange(H)) + emb_w = get_1d_sincos_pos_embed_from_grid(dim_w, pos=np.arange(W)) + emb_t = get_1d_sincos_pos_embed_from_grid(dim_t, pos=np.arange(T)) + + self.register_buffer("pos_emb_h", torch.from_numpy(emb_h).to(dtype=dtype, device=device), persistent=False) + self.register_buffer("pos_emb_w", torch.from_numpy(emb_w).to(dtype=dtype, device=device), persistent=False) + self.register_buffer("pos_emb_t", torch.from_numpy(emb_t).to(dtype=dtype, device=device), persistent=False) + self.pad_to_multiple_of = pad_to_multiple_of + + def forward( + self, + training_type: str | None = None, + ) -> torch.Tensor: + T, H, W = self.latent_shape + emb = torch.cat( + [ + repeat(self.pos_emb_t, "t d-> t h w d", h=H, w=W), + repeat(self.pos_emb_h, "h d-> t h w d", t=T, w=W), + repeat(self.pos_emb_w, "w d-> t h w d", t=T, h=H), + ], + dim=-1, + ) + # Flatten the T,H,W dimensions + emb = rearrange(emb, "t h w d -> (t h w) d") + + if training_type == "text_to_video": + bov_pe = torch.zeros((1, *emb.shape[1:]), device=emb.device, dtype=emb.dtype) + emb = torch.cat((bov_pe, emb), dim=0) + if self.pad_to_multiple_of is not None and emb.shape[0] % self.pad_to_multiple_of != 0: + pad_len = self.pad_to_multiple_of - emb.shape[0] % self.pad_to_multiple_of + emb = torch.cat((emb, torch.zeros((pad_len, *emb.shape[1:]), device=emb.device, dtype=emb.dtype)), dim=0) + seq_len, dim = emb.shape + emb = emb.reshape(1, seq_len, dim) + return emb diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/mlp.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/mlp.py new file mode 100644 index 00000000..75e28f25 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/mlp.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class MLP(nn.Module): + def __init__( + self, + dim: int, + hidden_dim: int, + ): + """ + Initializes the multilayer perceptron (MLP) module. + + Args: + dim: The input and output dimensionality. + hidden_dim: The dimensionality of the hidden layer. + """ + super().__init__() + self.w1 = nn.Linear(dim, hidden_dim, bias=False) + self.w2 = nn.Linear(hidden_dim, dim, bias=False) + self.w3 = nn.Linear(dim, hidden_dim, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Performs the forward pass of the MLP module. + + Args: + x: The input tensor of shape (batch_size, dim). + + Returns: + The output tensor of shape (batch_size, dim). + """ + output = self.w2(F.silu(self.w1(x)) * self.w3(x)) + return output diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/mm_projector.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/mm_projector.py new file mode 100644 index 00000000..ddb4d83f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/mm_projector.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""Multimodal projector to connect vision encoder / tokenizer with the LLM.""" + +from typing import Any, Optional + +import torch +import torch.nn as nn + + +class DownSampleBlock(nn.Module): + """Downsample block.""" + + def __init__(self): + super().__init__() + + def forward(self, x): + """ + Performs the forward pass of the downsample block. + + Args: + x (torch.Tensor): The input tensor from ViT's output of a sequence of embeddings. + Shape: (b, seq_len, c). + + Returns: + torch.Tensor: The output tensor. Shape: (b, seq_len/4, c*4). + """ + vit_embeds = x + # Get h and w as the sqrt of seq length. This assumes that the input is square-shaped. + h = w = int(vit_embeds.shape[1] ** 0.5) + b = vit_embeds.shape[0] + vit_embeds = vit_embeds.reshape(b, h, w, -1) + vit_embeds = self.flat_square(vit_embeds) + vit_embeds = vit_embeds.reshape(b, -1, vit_embeds.shape[-1]) + return vit_embeds + + def flat_square(self, x: torch.Tensor) -> torch.Tensor: + """ + Performs spatial downsampling while increasing the number of channels. + + Args: + x (torch.Tensor): The input tensor reshaped to a 2D grid. + Shape: (b, h, w, c) + + Returns: + torch.Tensor: The output tensor after the spatial downsampling. + Shape: (b, h/2, w/2, c*4) + """ + b, h, w, c = x.size() + # If w or h is odd, pad a column or a row of zeros. + if h % 2 == 1: + x = torch.concat([x, torch.zeros((b, 1, w, c), dtype=x.dtype).to(x.device)], dim=1).contiguous() + b, h, w, c = x.size() + if w % 2 == 1: + x = torch.concat([x, torch.zeros((b, h, 1, c), dtype=x.dtype).to(x.device)], dim=2).contiguous() + b, h, w, c = x.size() + # 2x spatial downsampling, 4x channel increasing. + x = x.view(b, h, int(w / 2), int(c * 2)) + x = x.permute(0, 2, 1, 3).contiguous() + x = x.view(b, int(h / 2), int(w / 2), int(c * 4)) + x = x.permute(0, 2, 1, 3).contiguous() + return x + + +class MultimodalProjector(nn.Module): + """Multimodal projector.""" + + def __init__( + self, + mm_projector_type: str, + in_dim: int, + out_dim: Optional[int] = None, + **kwargs: Any, + ): + super().__init__() + if out_dim is None: + out_dim = in_dim + if mm_projector_type == "identity": + self.projector = nn.Identity() + elif mm_projector_type == "linear": + self.projector = nn.Linear(in_dim, out_dim) + elif mm_projector_type == "mlp": + self.projector = nn.Sequential(nn.Linear(in_dim, out_dim), nn.GELU(), nn.Linear(out_dim, out_dim)) + elif mm_projector_type == "mlp_downsample": + self.projector = nn.Sequential( + DownSampleBlock(), + nn.LayerNorm(in_dim * 4), + nn.Linear(in_dim * 4, out_dim), + nn.GELU(), + nn.Linear(out_dim, out_dim), + ) + else: + raise ValueError(f"Unknown projector type: {mm_projector_type}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.projector(x) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/normalization.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/normalization.py new file mode 100644 index 00000000..0e843112 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/modules/normalization.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +import torch.nn as nn + + +def create_norm(norm_type: str, dim: int, eps: float = 1e-6): + """ + Creates the specified normalization layer based on the norm_type. + Adopted from TorchTriton: https://github.com/pytorch/torchtitan/blob/main/torchtitan/models/norms.py + + Args: + norm_type (str): The type of normalization layer to create. + Supported types: 1. rmsnorm 2. fused_rmsnorm 3. layernorm 4. np_layernorm + dim (int): The dimension of the normalization layer. + eps (float, optional): The epsilon value for numerical stability. Defaults to 1e-6. + + Returns: + The created normalization layer. + + Raises: + NotImplementedError: If an unknown norm_type is provided. + """ + norm_type = norm_type.lower() # Normalize to lowercase + + if norm_type == "layernorm": + return nn.LayerNorm(dim, eps=eps, bias=False) + elif norm_type == "np_layernorm": + return nn.LayerNorm(dim, eps=eps, elementwise_affine=False, bias=False) + elif norm_type == "rmsnorm": + return RMSNorm(dim, eps=eps, compile=False) + elif norm_type == "compiled_rmsnorm": + return RMSNorm(dim, eps=eps, compile=True) + elif norm_type == "fused_rmsnorm": + raise NotImplementedError("Fused RMSNorm is not supported yet.") + else: + raise NotImplementedError(f"Unknown norm_type: '{norm_type}'") + + +class RMSNorm(nn.Module): + """ + Initialize the RMSNorm normalization layer. + Reference implementation: https://github.com/pytorch/torchtitan/blob/main/torchtitan/models/norms.py + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + compile (bool, optional): Whether to compile the forward function. Default is False. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + + def __init__(self, dim: int, eps: float = 1e-6, compile: bool = False): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + self.rmsnorm_fn = torch.compile(self.compute_rmsnorm, fullgraph=True) if compile else self.compute_rmsnorm + + @staticmethod + def compute_rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float): + def _norm(x, eps): + # Computes the root-mean-square norm of the input tensor. + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) + + output = _norm(x.float(), eps).type_as(x) + return output * weight + + def forward(self, x: torch.Tensor): + return self.rmsnorm_fn(x, self.weight, self.eps) + + def reset_parameters(self): + torch.nn.init.ones_(self.weight) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos.py new file mode 100644 index 00000000..dd2bf92c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos.py @@ -0,0 +1,374 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Callable, Optional + +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from megatron.core.models.common.embeddings.rotary_pos_embedding import RotaryEmbedding +from megatron.core.models.gpt.gpt_model import GPTModel as MCoreGPTModel +from megatron.core.transformer.enums import AttnBackend +from nemo.collections.llm.gpt.model.llama import Llama3Config, LlamaModel +from nemo.collections.llm.utils import Config +from nemo.lightning import OptimizerModule, io +from nemo.lightning.base import teardown +from torch import Tensor, nn + +from cosmos1.utils import log + + +class RotaryEmbedding3D(RotaryEmbedding): + """Rotary Embedding3D for Cosmos Language model. + Args: + kv_channels (int): Projection weights dimension in multi-head attention. Obtained + from transformer config + rotary_base (int, optional): Base period for rotary position embeddings. Defaults to + 10000. + use_cpu_initialization (bool, optional): If False, initialize the inv_freq directly + on the GPU. Defaults to False + latent_shape: The shape of the latents produced by the video after being tokenized + """ + + def __init__( + self, + seq_len: int, + kv_channels: int, + training_type: str | None = None, + rotary_base: int = 10000, + use_cpu_initialization: bool = False, + latent_shape=[5, 40, 64], + apply_yarn=False, + original_latent_shape=None, + beta_fast=32, + beta_slow=1, + scale=None, + max_position_embeddings=None, + original_max_position_embeddings=None, + extrapolation_factor=1, + attn_factor=1, + pad_to_multiple_of=None, + ) -> None: + super().__init__( + kv_channels=kv_channels, + rotary_base=rotary_base, + rotary_percent=1.0, + use_cpu_initialization=use_cpu_initialization, + ) + self.training_type = training_type + self.latent_shape = latent_shape + self.device = "cpu" if use_cpu_initialization else torch.cuda.current_device() + self.dim = kv_channels + self.rope_theta = rotary_base + self.apply_yarn = apply_yarn + self.original_latent_shape = original_latent_shape + self.beta_fast = beta_fast + self.beta_slow = beta_slow + self.scale = scale + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.attn_factor = attn_factor + dim_h = self.dim // 6 * 2 + dim_t = self.dim - 2 * dim_h + self.dim_spatial_range = torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().to(self.device) / dim_h + spatial_inv_freq = 1.0 / (self.rope_theta**self.dim_spatial_range) + self.dim_temporal_range = torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().to(self.device) / dim_t + temporal_inv_freq = 1.0 / (self.rope_theta**self.dim_temporal_range) + if self.apply_yarn: + assert self.original_latent_shape is not None, "Original latent shape required." + assert self.beta_slow is not None, "Beta slow value required." + assert self.beta_fast is not None, "Beta fast value required." + scale_factors_spatial = self.get_scale_factors(spatial_inv_freq, self.original_latent_shape[1]) + spatial_inv_freq = spatial_inv_freq * scale_factors_spatial + scale_factors_temporal = self.get_scale_factors(temporal_inv_freq, self.original_latent_shape[0]) + temporal_inv_freq = temporal_inv_freq * scale_factors_temporal + self.mscale = float(self.get_mscale(self.scale) * self.attn_factor) + self.spatial_inv_freq = spatial_inv_freq + self.temporal_inv_freq = temporal_inv_freq + max_seq_len_cached = max(self.latent_shape) + if self.apply_yarn and seq_len > max_seq_len_cached: + max_seq_len_cached = seq_len + self.max_seq_len_cached = max_seq_len_cached + self.pad_to_multiple_of = pad_to_multiple_of + self.freqs = self.get_freqs_non_repeated(self.max_seq_len_cached) + + def get_mscale(self, scale: float = 1.0) -> float: + """Get the magnitude scaling factor for YaRN.""" + if scale <= 1: + return 1.0 + return 0.1 * math.log(scale) + 1.0 + + def get_scale_factors(self, inv_freq: torch.Tensor, original_seq_len: int) -> torch.Tensor: + """Get the scale factors for YaRN.""" + # Calculate the high and low frequency cutoffs for YaRN. Note: `beta_fast` and `beta_slow` are called + # `high_freq_factor` and `low_freq_factor` in the Llama 3.1 RoPE scaling code. + high_freq_cutoff = 2 * math.pi * self.beta_fast / original_seq_len + low_freq_cutoff = 2 * math.pi * self.beta_slow / original_seq_len + # Obtain a smooth mask that has a value of 0 for low frequencies and 1 for high frequencies, with linear + # interpolation in between. + smooth_mask = torch.clamp((inv_freq - low_freq_cutoff) / (high_freq_cutoff - low_freq_cutoff), min=0, max=1) + # For low frequencies, we scale the frequency by 1/self.scale. For high frequencies, we keep the frequency. + scale_factors = (1 - smooth_mask) / self.scale + smooth_mask + return scale_factors + + def get_freqs_non_repeated(self, max_seq_len_cached: int, offset: int = 0) -> Tensor: + dtype = self.spatial_inv_freq.dtype + device = self.spatial_inv_freq.device + + self.seq = (torch.arange(max_seq_len_cached, device=device, dtype=dtype) + offset).cuda() + + assert hasattr(self, "latent_shape"), ( + "Latent shape is not set. Please run set_latent_shape() method on rope embedding. " + ) + T, H, W = self.latent_shape + half_emb_t = torch.outer(self.seq[:T], self.temporal_inv_freq.cuda()) + half_emb_h = torch.outer(self.seq[:H], self.spatial_inv_freq.cuda()) + half_emb_w = torch.outer(self.seq[:W], self.spatial_inv_freq.cuda()) + emb = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + emb = rearrange(emb, "t h w d -> (t h w) 1 1 d").float() + + if self.training_type == "text_to_video": + bov_pe = torch.zeros((1, *emb.shape[1:]), device=emb.device) + emb = torch.cat((bov_pe, emb), dim=0) + + if self.pad_to_multiple_of is not None and emb.shape[0] % self.pad_to_multiple_of != 0: + # Round up to the nearest multiple of pad_to_multiple_of + pad_len = self.pad_to_multiple_of - emb.shape[0] % self.pad_to_multiple_of + emb = torch.cat((emb, torch.zeros((pad_len, *emb.shape[1:]), device=emb.device)), dim=0) + + return emb + + @lru_cache(maxsize=32) + def forward(self, seq_len: int, offset: int = 0, packed_seq: bool = False) -> Tensor: + if self.spatial_inv_freq.device.type == "cpu": + # move `inv_freq` to GPU once at the first micro-batch forward pass + self.spatial_inv_freq = self.spatial_inv_freq.to(device=torch.cuda.current_device()) + + max_seq_len_cached = self.max_seq_len_cached + if self.apply_yarn and seq_len > max_seq_len_cached: + max_seq_len_cached = seq_len + self.max_seq_len_cached = max_seq_len_cached + emb = self.get_freqs_non_repeated(self.max_seq_len_cached) + return emb + + +if TYPE_CHECKING: + from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec + + +@dataclass +class CosmosConfig(Llama3Config): + qk_layernorm: bool = True + rope_dim: str = "3D" + vocab_size: int = 64000 + activation_func = F.silu + attention_backend: AttnBackend = AttnBackend.flash + + def configure_model(self, tokenizer) -> "MCoreGPTModel": + model = super().configure_model(tokenizer) + if self.rope_dim == "3D": + model.rotary_pos_emb = RotaryEmbedding3D( + seq_len=self.seq_length, + training_type=None, + kv_channels=self.kv_channels, + max_position_embeddings=self.seq_length, + original_max_position_embeddings=self.original_seq_len if hasattr(self, "original_seq_len") else None, + rotary_base=self.rotary_base, + apply_yarn=True if hasattr(self, "apply_yarn") else False, + scale=self.yarn_scale if hasattr(self, "yarn_scale") else None, + extrapolation_factor=1, + attn_factor=1, + beta_fast=self.yarn_beta_fast if hasattr(self, "yarn_beta_fast") else 32, + beta_slow=self.yarn_beta_slow if hasattr(self, "yarn_beta_slow") else 1, + latent_shape=[5, 40, 64], + original_latent_shape=self.original_latent_shape if hasattr(self, "original_latent_shape") else None, + ) + return model + + +@dataclass +class CosmosConfig4B(CosmosConfig): + rotary_base: int = 500_000 + seq_length: int = 15360 + num_layers: int = 16 + hidden_size: int = 4096 + ffn_hidden_size: int = 14336 + num_attention_heads: int = 32 + num_query_groups: int = 8 + layernorm_epsilon: float = 1e-5 + use_cpu_initialization: bool = True + make_vocab_size_divisible_by: int = 128 + kv_channels: int = 128 + + +@dataclass +class CosmosConfig12B(CosmosConfig): + rotary_base: int = 500_000 + seq_length: int = 15360 + num_layers: int = 40 + hidden_size: int = 5120 + ffn_hidden_size: int = 14336 + num_attention_heads: int = 32 + num_query_groups: int = 8 + layernorm_epsilon: float = 1e-5 + use_cpu_initialization: bool = True + make_vocab_size_divisible_by: int = 128 + kv_channels: int = 128 + original_latent_shape = [3, 40, 64] + apply_yarn: bool = True + yarn_beta_fast: int = 4 + yarn_beta_slow: int = 1 + yarn_scale: int = 2 + original_seq_len = 8192 + + +class CosmosModel(LlamaModel): + def __init__( + self, + config: Annotated[Optional[CosmosConfig], Config[CosmosConfig]] = None, + optim: Optional[OptimizerModule] = None, + tokenizer: Optional["TokenizerSpec"] = None, + model_transform: Optional[Callable[[nn.Module], nn.Module]] = None, + ): + super().__init__(config or CosmosConfig4B(), optim=optim, tokenizer=tokenizer, model_transform=model_transform) + self.config = config + + +@io.state_transform( + source_key=( + "model.layers.*.feed_forward.w1.weight", + "model.layers.*.feed_forward.w3.weight", + ), + target_key="decoder.layers.*.mlp.linear_fc1.weight", +) +def _mlp_glu(ctx: io.TransformCTX, w1, w3): + return torch.cat((w1, w3), axis=0) + + +@io.state_transform( + source_key=( + "model.layers.*.attention.wq.weight", + "model.layers.*.attention.wk.weight", + "model.layers.*.attention.wv.weight", + ), + target_key="decoder.layers.*.self_attention.linear_qkv.weight", +) +def _import_qkv_cosmos(ctx: io.TransformCTX, q, k, v): + megatron_config = ctx.target.config + + head_num = megatron_config.num_attention_heads + num_query_groups = megatron_config.num_query_groups + heads_per_group = head_num // num_query_groups + hidden_size = megatron_config.hidden_size + head_size = megatron_config.kv_channels + + old_tensor_shape = q.size() + new_q_tensor_shape = (head_num, head_size) + old_tensor_shape[1:] + new_kv_tensor_shape = (num_query_groups, head_size) + old_tensor_shape[1:] + + q = q.view(*new_q_tensor_shape) + k = k.view(*new_kv_tensor_shape) + v = v.view(*new_kv_tensor_shape) + + qkv_weights_l = [] + for i in range(num_query_groups): + qkv_weights_l.append(q[i * heads_per_group : (i + 1) * heads_per_group, :, :]) + qkv_weights_l.append(k[i : i + 1, :, :]) + qkv_weights_l.append(v[i : i + 1, :, :]) + qkv_weights = torch.cat(qkv_weights_l) + assert qkv_weights.ndim == 3, qkv_weights.shape + assert qkv_weights.shape[0] == (heads_per_group + 2) * num_query_groups, qkv_weights.shape + assert qkv_weights.shape[1] == head_size, qkv_weights.shape + assert qkv_weights.shape[2] == old_tensor_shape[1], qkv_weights.shape + + qkv_weights = qkv_weights.reshape([head_size * (head_num + 2 * num_query_groups), hidden_size]) + + return qkv_weights + + +@io.model_importer(CosmosModel, "pt") +class PTCosmosImporter(io.ModelConnector["PTCosmosModel", CosmosModel]): + def init(self) -> CosmosModel: + return CosmosModel(self.config, tokenizer=self.tokenizer) + + def apply(self, output_path: Path) -> Path: + pt_model_path = str(self) + cosmos_model_state_dict = torch.load(pt_model_path, map_location="cpu") + for k, v in cosmos_model_state_dict.items(): + # convert to float 32 (for cpu conversion) (Original model is bf16) + cosmos_model_state_dict[k] = v.float() + + # Small wrapper since nemo calls source.state_dict() , to get state dict + class WrapperCosmos: + def __init__(self, model_state_dict): + self.model_state_dict = model_state_dict + + def state_dict(self): + return self.model_state_dict + + source = WrapperCosmos(cosmos_model_state_dict) + target = self.init() + trainer = self.nemo_setup(target) + self.convert_state(source, target) + self.nemo_save(output_path, trainer) + + log.info(f"Converted PT Cosmos model to Nemo, model saved to {output_path}") + + teardown(trainer, target) + del trainer, target + + return output_path + + def convert_state(self, source, target): + mapping = { + "model.tok_embeddings.weight": "embedding.word_embeddings.weight", + "model.layers.*.attention.wo.weight": "decoder.layers.*.self_attention.linear_proj.weight", + "model.layers.*.attention.q_norm.weight": "decoder.layers.*.self_attention.q_layernorm.weight", + "model.layers.*.attention.k_norm.weight": "decoder.layers.*.self_attention.k_layernorm.weight", + "model.layers.*.attention_norm.weight": "decoder.layers.*.self_attention.linear_qkv.layer_norm_weight", + "model.layers.*.feed_forward.w2.weight": "decoder.layers.*.mlp.linear_fc2.weight", + "model.layers.*.ffn_norm.weight": "decoder.layers.*.mlp.linear_fc1.layer_norm_weight", + "model.norm.weight": "decoder.final_layernorm.weight", + "model.output.weight": "output_layer.weight", + } + + return io.apply_transforms(source, target, mapping=mapping, transforms=[_import_qkv_cosmos, _mlp_glu]) + + @property + def tokenizer(self): + return None + + @property + def config(self): + if "4B" in str(self) or "4b" in str(self): + return CosmosConfig4B() + elif "12B" in str(self) or "12b" in str(self): + return CosmosConfig12B() + else: + raise ValueError("Unable to infer model size from checkpoint") diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos_action_control.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos_action_control.py new file mode 100644 index 00000000..a56369ac --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos_action_control.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from dataclasses import dataclass +from typing import Callable, Optional + +import torch +import torch.nn as nn +from megatron.core import InferenceParams +from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import InferenceWrapperConfig +from megatron.core.models.gpt.gpt_model import GPTModel as MCoreGPTModel + +from cosmos1.models.autoregressive.nemo.cosmos import CosmosConfig4B, CosmosConfig12B +from cosmos1.models.autoregressive.nemo.cosmos_video2world import CosmosVideo2WorldConfig, CosmosVideo2WorldModel +from cosmos1.models.autoregressive.nemo.inference.inference_controller import CosmosActionControlInferenceWrapper + + +@dataclass +class CosmosActionControlConfig(CosmosVideo2WorldConfig): + action_dim: int = 7 + forward_step_fn: Callable = lambda model, batch: model(**batch) + latent_shape: tuple[int, int, int] = (2, 30, 40) + pad_to_multiple_of: Optional[int] = 128 + seq_length: int = 2_432 + + def configure_model(self, tokenizer) -> "MCoreGPTModel": + model = super().configure_model(tokenizer) + + model.action_mlp = nn.Sequential( + nn.Linear(self.action_dim, self.crossattn_emb_size // 4), + nn.ReLU(), + nn.Linear(self.crossattn_emb_size // 4, self.crossattn_emb_size), + ) + return model + + +class CosmosActionControlModel(CosmosVideo2WorldModel): + def forward( + self, + tokens: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor, + labels: torch.Tensor, + abs_pos_embed: torch.Tensor, + action: torch.Tensor, + loss_mask: torch.Tensor | None = None, + inference_params: InferenceParams | None = None, + ) -> torch.Tensor: + # Apply the action MLP to the action to generate the context vector + mcore_model = self + while not isinstance(mcore_model, MCoreGPTModel) and hasattr(mcore_model, "module"): + mcore_model = mcore_model.module + + action_mlp_out = mcore_model.action_mlp(action) + context = action_mlp_out.unsqueeze(0).repeat(self.config.action_dim, 1, 1) + + return super().forward( + input_ids=tokens, + position_ids=position_ids, + attention_mask=attention_mask, + labels=labels, + extra_block_kwargs={ + "context": context, + "extra_positional_embeddings": abs_pos_embed, + }, + packed_seq_params=None, + inference_params=inference_params, + ) + + def get_inference_wrapper(self, params_dtype, inference_batch_times_seqlen_threshold) -> torch.Tensor: + # This is to get the MCore model required in GPTInferenceWrapper. + mcore_model = self.module + vocab_size = self.config.vocab_size + + inference_wrapper_config = InferenceWrapperConfig( + hidden_size=mcore_model.config.hidden_size, + params_dtype=params_dtype, + inference_batch_times_seqlen_threshold=inference_batch_times_seqlen_threshold, + padded_vocab_size=vocab_size, + ) + + model_inference_wrapper = CosmosActionControlInferenceWrapper( + mcore_model, inference_wrapper_config, self.config + ) + return model_inference_wrapper + + +@dataclass +class CosmosConfigActionControl5B(CosmosActionControlConfig, CosmosConfig4B): + make_vocab_size_divisible_by: int = 64 + + +@dataclass +class CosmosConfigActionControl13B(CosmosActionControlConfig, CosmosConfig12B): + make_vocab_size_divisible_by: int = 128 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos_video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos_video2world.py new file mode 100644 index 00000000..31d3e368 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/cosmos_video2world.py @@ -0,0 +1,668 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import copy +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Callable, Dict, Optional, Union + +import torch +from megatron.core import tensor_parallel +from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add +from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import InferenceWrapperConfig +from megatron.core.models.gpt.gpt_model import GPTModel as MCoreGPTModel +from megatron.core.transformer.attention import ( + Attention, + CrossAttentionSubmodules, + SelfAttention, + SelfAttentionSubmodules, +) +from megatron.core.transformer.custom_layers.transformer_engine import ( + TEColumnParallelLinear, + TEDotProductAttention, + TELayerNormColumnParallelLinear, + TENorm, + TERowParallelLinear, +) +from megatron.core.transformer.enums import AttnBackend, AttnMaskType +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from megatron.core.utils import make_viewless_tensor +from torch import nn + +from cosmos1.models.autoregressive.nemo.cosmos import ( + CosmosConfig, + CosmosConfig4B, + CosmosConfig12B, + CosmosModel, + RotaryEmbedding3D, +) +from cosmos1.models.autoregressive.nemo.inference.inference_controller import CosmosInferenceWrapper +from cosmos1.utils import log + + +if TYPE_CHECKING: + from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec + +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.transformer_block import TransformerBlock +from nemo.collections.llm.gpt.model.base import get_batch_on_this_context_parallel_rank +from nemo.collections.llm.utils import Config +from nemo.lightning import OptimizerModule, io +from nemo.lightning.base import teardown + + +class CosmosTransformerBlock(TransformerBlock): + def forward( + self, + *args, + packed_seq_params: PackedSeqParams = None, + extra_positional_embeddings=None, + **kwargs, + ): + packed_seq_params = {"abs_pos_embed": extra_positional_embeddings} + return super().forward( + *args, + packed_seq_params=packed_seq_params, + **kwargs, + ) + + +@dataclass +class CosmosCrossAttentionSubmodules(CrossAttentionSubmodules): + q_layernorm: Union[ModuleSpec, type] = None + k_layernorm: Union[ModuleSpec, type] = None + + +class CrossAttention(Attention): + """Cross-attention layer class + + Cross-attention layer takes input with size [s, b, h] and context with size + [s, b, h] and returns output of the same size. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: CosmosCrossAttentionSubmodules, + layer_number: int, + attn_mask_type=AttnMaskType.padding, + ): + config = copy.deepcopy(config) + # config.num_query_groups = config.num_attention_heads + + super().__init__( + config=config, + submodules=submodules, + layer_number=layer_number, + attn_mask_type=attn_mask_type, + attention_type="cross", + ) + + self.linear_q = build_module( + submodules.linear_q, + self.config.hidden_size, + self.query_projection_size, + config=self.config, + init_method=self.config.init_method, + gather_output=False, + bias=self.config.add_bias_linear, + skip_bias_add=False, + is_expert=False, + ) + + self.linear_kv = build_module( + submodules.linear_kv, + self.config.crossattn_emb_size, + 2 * self.kv_projection_size, + config=self.config, + init_method=self.config.init_method, + gather_output=False, + bias=self.config.add_bias_linear, + skip_bias_add=False, + is_expert=False, + ) + + self.q_layernorm = build_module( + submodules.q_layernorm, + hidden_size=self.hidden_size_per_attention_head, + config=self.config, + eps=self.config.layernorm_epsilon, + ) + + self.k_layernorm = build_module( + submodules.k_layernorm, + hidden_size=self.hidden_size_per_attention_head, + config=self.config, + eps=self.config.layernorm_epsilon, + ) + + def get_query_key_value_tensors(self, hidden_states, key_value_states): + """ + Derives `query` tensor from `hidden_states`, and `key`/`value` tensors + from `key_value_states`. + """ + # Attention heads [sk, b, h] --> [sk, b, (np * 2 * hn)] + mixed_kv, _ = self.linear_kv(key_value_states) + + # [sk, b, (np * 2 * hn)] --> [sk, b, np, 2 * hn] + new_tensor_shape = mixed_kv.size()[:-1] + ( + self.num_query_groups_per_partition, + 2 * self.hidden_size_per_attention_head, + ) + mixed_kv = mixed_kv.view(*new_tensor_shape) + + # [sk, b, np, 2 * hn] --> 2 [sk, b, np, hn] + (key, value) = tensor_parallel.split_tensor_along_last_dim(mixed_kv, 2) + + # Attention head [sq, b, h] --> [sq, b, hp] + query, _ = self.linear_q(hidden_states) + + # [sq, b, hp] --> [sq, b, np, hn] + new_tensor_shape = query.size()[:-1] + ( + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + ) + query = query.view(*new_tensor_shape) + + if self.q_layernorm is not None: + query = self.q_layernorm(query) + + if self.k_layernorm is not None: + key = self.k_layernorm(key) + + return query, key, value + + +class CosmosVideo2WorldTransformerLayer(TransformerLayer): + def __init__( + self, + config: TransformerConfig, + submodules: TransformerLayerSubmodules, + layer_number: int = 1, + hidden_dropout: float = None, + ): + super().__init__( + config=config, submodules=submodules, layer_number=layer_number, hidden_dropout=hidden_dropout + ) + + def forward( + self, + hidden_states, + attention_mask=None, + context=None, + context_mask=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=None, + **kwargs, + ): + assert "abs_pos_embed" in packed_seq_params + abs_pos_embed = packed_seq_params["abs_pos_embed"] + assert abs_pos_embed.shape[0] == hidden_states.shape[0], ( + f"Abs pos embed shape : {abs_pos_embed.shape}, hidden states shape : {hidden_states.shape}. They should match at the zeroth dimension" + ) + packed_seq_params = None + + hidden_states = abs_pos_embed + hidden_states + + rotary_pos_emb + residual = hidden_states + + # Optional Input Layer norm + input_layernorm_output = self.input_layernorm(hidden_states) + + # Self attention. + attention_output_with_bias = self.self_attention( + input_layernorm_output, + attention_mask=attention_mask, + inference_params=inference_params, + rotary_pos_emb=rotary_pos_emb, + packed_seq_params=packed_seq_params, + ) + + with self.bias_dropout_add_exec_handler(): + hidden_states = self.self_attn_bda(self.training, self.config.bias_dropout_fusion)( + attention_output_with_bias, residual, self.hidden_dropout + ) + + # Residual connection. + residual = hidden_states + + # Optional Layer norm after self-attention + pre_cross_attn_layernorm_output = self.pre_cross_attn_layernorm(hidden_states) + + attention_output_with_bias = self.cross_attention( + pre_cross_attn_layernorm_output, + attention_mask=context_mask, + key_value_states=context, + inference_params=None, + ) + # print(f'After attn : {attention_output_with_bias[0].sum().item()}') + + if isinstance(attention_output_with_bias, dict) and "context" in attention_output_with_bias: + context = attention_output_with_bias["context"] + + with self.bias_dropout_add_exec_handler(): + hidden_states = self.cross_attn_bda(self.training, self.config.bias_dropout_fusion)( + attention_output_with_bias, residual, self.hidden_dropout + ) + + # Residual connection. + residual = hidden_states + + # Optional Layer norm post the cross-attention. + pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) + + # MLP. + mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) + # print(f'After ffn : {mlp_output_with_bias[0].sum().item()}') + + with self.bias_dropout_add_exec_handler(): + hidden_states = self.mlp_bda(self.training, self.config.bias_dropout_fusion)( + mlp_output_with_bias, residual, self.hidden_dropout + ) + + # Jit compiled function creates 'view' tensor. This tensor + # potentially gets saved in the MPU checkpoint function context, + # which rejects view tensors. While making a viewless tensor here + # won't result in memory savings (like the data loader, or + # p2p_communication), it serves to document the origin of this + # 'view' tensor. + # print(f'Final out : {hidden_states.sum().item()}') + output = make_viewless_tensor(inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True) + # CUDA graph requires returned values to be Tensors + if self.config.external_cuda_graph and self.training: + return output + return output, context + + +def get_cosmos_video2world_spec() -> ModuleSpec: + return ModuleSpec( + module=CosmosVideo2WorldTransformerLayer, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=TELayerNormColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=TENorm, + k_layernorm=TENorm, + ), + ), + self_attn_bda=get_bias_dropout_add, + pre_cross_attn_layernorm=TENorm, + cross_attention=ModuleSpec( + module=CrossAttention, + params={"attn_mask_type": AttnMaskType.no_mask}, + submodules=CosmosCrossAttentionSubmodules( + linear_q=TEColumnParallelLinear, + linear_kv=TEColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + q_layernorm=TENorm, + k_layernorm=TENorm, + ), + ), + cross_attn_bda=get_bias_dropout_add, + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TELayerNormColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + mlp_bda=get_bias_dropout_add, + ), + ) + + +def cosmos_data_step(dataloader_iter) -> Dict[str, torch.Tensor]: + from megatron.core import parallel_state + + # Based on: https://github.com/NVIDIA/Megatron-LM/blob/main/pretrain_gpt.py#L87 + # https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py#L828-L842 + + batch = next(dataloader_iter) + + _batch: dict + if isinstance(batch, tuple) and len(batch) == 3: + _batch = batch[0] + else: + _batch = batch + + required_device_keys = set() + required_host_keys = set() + + required_device_keys.add("attention_mask") + if "cu_seqlens" in _batch: + raise ValueError("Packed sequence cu_seqlens not supported") + + required_device_keys.update(("context", "abs_pos_embed", "action")) + if parallel_state.is_pipeline_first_stage(): + required_device_keys.update(("tokens", "position_ids")) + if parallel_state.is_pipeline_last_stage(): + required_device_keys.update(("labels", "loss_mask")) + + _batch_required_keys = {} + for key, val in _batch.items(): + if key in required_device_keys: + _batch_required_keys[key] = val.cuda(non_blocking=True) + elif key in required_host_keys: + _batch_required_keys[key] = val.cpu() + else: + _batch_required_keys[key] = None + + # slice batch along sequence dimension for context parallelism + output = get_batch_on_this_context_parallel_rank(_batch_required_keys) + + return output + + +def cosmos_forward_step(model, batch) -> torch.Tensor: + forward_args = {} + forward_args["input_ids"] = batch["tokens"] + forward_args["position_ids"] = batch["position_ids"] + forward_args["attention_mask"] = batch["attention_mask"] + forward_args["labels"] = batch["labels"] + forward_args["extra_block_kwargs"] = { + "context": batch["context"], + "extra_positional_embeddings": batch["abs_pos_embed"], + } + forward_args["packed_seq_params"] = None + + return model(**forward_args) + + +@dataclass +class CosmosVideo2WorldConfig: + vocab_size: int = 64064 + output_layer_vocab_size: int = 64000 + seq_length: int = 12864 + latent_shape = [5, 40, 64] + pad_to_multiple_of = 64 + forward_step_fn: Callable = cosmos_forward_step + transformer_layer_spec = get_cosmos_video2world_spec() + data_step_fn: Callable = cosmos_data_step + attention_backend: AttnBackend = AttnBackend.flash + crossattn_emb_size: int = 1024 + kv_channels: int = 128 + training_type: str | None = "text_to_video" + + def configure_model(self, tokenizer) -> "MCoreGPTModel": + self.transformer_layer_spec = get_cosmos_video2world_spec() + model = super().configure_model(tokenizer) + if self.rope_dim == "3D": + model.rotary_pos_emb = RotaryEmbedding3D( + seq_len=self.seq_length, + training_type=self.training_type, + pad_to_multiple_of=self.pad_to_multiple_of, + kv_channels=self.kv_channels, + max_position_embeddings=self.seq_length, + original_max_position_embeddings=self.original_seq_len if hasattr(self, "original_seq_len") else None, + rotary_base=self.rotary_base, + apply_yarn=True if hasattr(self, "apply_yarn") else False, + scale=self.yarn_scale if hasattr(self, "yarn_scale") else None, + extrapolation_factor=1, + attn_factor=1, + beta_fast=self.yarn_beta_fast if hasattr(self, "yarn_beta_fast") else 32, + beta_slow=self.yarn_beta_slow if hasattr(self, "yarn_beta_slow") else 1, + latent_shape=self.latent_shape, + original_latent_shape=self.original_latent_shape if hasattr(self, "original_latent_shape") else None, + ) + model.output_layer = tensor_parallel.ColumnParallelLinear( + self.hidden_size, + self.output_layer_vocab_size, + config=self, + init_method=self.init_method, + bias=False, + skip_bias_add=False, + gather_output=False, + skip_weight_param_allocation=False, + embedding_activation_buffer=None, + grad_output_buffer=None, + ) + + model.decoder = CosmosTransformerBlock( + config=self, + spec=self.transformer_layer_spec, + pre_process=model.pre_process, + post_process=model.post_process, + ) + return model + + +@dataclass +class CosmosConfigVideo2World5B(CosmosVideo2WorldConfig, CosmosConfig4B): + make_vocab_size_divisible_by: int = 64 + + +@dataclass +class CosmosConfigVideo2World13B(CosmosVideo2WorldConfig, CosmosConfig12B): + make_vocab_size_divisible_by: int = 128 + + +class CosmosVideo2WorldModel(CosmosModel): + def __init__( + self, + config: Annotated[Optional[CosmosConfig], Config[CosmosConfig]] = None, + optim: Optional[OptimizerModule] = None, + tokenizer: Optional["TokenizerSpec"] = None, + model_transform: Optional[Callable[[nn.Module], nn.Module]] = None, + ): + super().__init__( + config or CosmosConfigVideo2World5B(), optim=optim, tokenizer=tokenizer, model_transform=model_transform + ) + self.config = config + + def get_inference_wrapper(self, params_dtype, inference_batch_times_seqlen_threshold) -> torch.Tensor: + # This is to get the MCore model required in GPTInferenceWrapper. + mcore_model = self.module + + vocab_size = self.config.vocab_size + + inference_wrapper_config = InferenceWrapperConfig( + hidden_size=mcore_model.config.hidden_size, + params_dtype=params_dtype, + inference_batch_times_seqlen_threshold=inference_batch_times_seqlen_threshold, + padded_vocab_size=vocab_size, + ) + + model_inference_wrapper = CosmosInferenceWrapper(mcore_model, inference_wrapper_config, self.config) + return model_inference_wrapper + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + decoder_input: Optional[torch.Tensor] = None, + inference_params=None, + packed_seq_params=None, + extra_block_kwargs=None, + ) -> torch.Tensor: + extra_kwargs = {"packed_seq_params": packed_seq_params} if packed_seq_params is not None else {} + output_tensor = self.module( + input_ids, + position_ids, + attention_mask, + decoder_input=decoder_input, + labels=labels, + inference_params=inference_params, + extra_block_kwargs=extra_block_kwargs, + **extra_kwargs, + ) + + return output_tensor + + +@io.state_transform( + source_key=( + "model.layers.*.feed_forward.w1.weight", + "model.layers.*.feed_forward.w3.weight", + ), + target_key="decoder.layers.*.mlp.linear_fc1.weight", +) +def _mlp_glu(ctx: io.TransformCTX, w1, w3): + return torch.cat((w1, w3), axis=0) + + +@io.state_transform( + source_key=( + "model.layers.*.attention.wq.weight", + "model.layers.*.attention.wk.weight", + "model.layers.*.attention.wv.weight", + ), + target_key="decoder.layers.*.self_attention.linear_qkv.weight", +) +def _import_qkv_cosmos_self_attention(ctx: io.TransformCTX, q, k, v): + megatron_config = ctx.target.config + + head_num = megatron_config.num_attention_heads + num_query_groups = megatron_config.num_query_groups + heads_per_group = head_num // num_query_groups + hidden_size = megatron_config.hidden_size + head_size = megatron_config.kv_channels + + old_tensor_shape = q.size() + new_q_tensor_shape = (head_num, head_size) + old_tensor_shape[1:] + new_kv_tensor_shape = (num_query_groups, head_size) + old_tensor_shape[1:] + + q = q.view(*new_q_tensor_shape) + k = k.view(*new_kv_tensor_shape) + v = v.view(*new_kv_tensor_shape) + + qkv_weights_l = [] + for i in range(num_query_groups): + qkv_weights_l.append(q[i * heads_per_group : (i + 1) * heads_per_group, :, :]) + qkv_weights_l.append(k[i : i + 1, :, :]) + qkv_weights_l.append(v[i : i + 1, :, :]) + qkv_weights = torch.cat(qkv_weights_l) + assert qkv_weights.ndim == 3, qkv_weights.shape + assert qkv_weights.shape[0] == (heads_per_group + 2) * num_query_groups, qkv_weights.shape + assert qkv_weights.shape[1] == head_size, qkv_weights.shape + assert qkv_weights.shape[2] == old_tensor_shape[1], qkv_weights.shape + + qkv_weights = qkv_weights.reshape([head_size * (head_num + 2 * num_query_groups), hidden_size]) + + return qkv_weights + + +@io.state_transform( + source_key=( + "model.layers.*.cross_attention.wk.weight", + "model.layers.*.cross_attention.wv.weight", + ), + target_key="decoder.layers.*.cross_attention.linear_kv.weight", +) +def _import_kv_cosmos_cross_attention(ctx: io.TransformCTX, k, v): + megatron_config = ctx.target.config + + num_query_groups = megatron_config.num_query_groups + hidden_size = megatron_config.crossattn_emb_size + head_size = megatron_config.kv_channels + new_kv_tensor_shape = (num_query_groups, head_size) + (hidden_size,) + k = k.view(*new_kv_tensor_shape) + v = v.view(*new_kv_tensor_shape) + + kv_weights_l = [] + for i in range(num_query_groups): + kv_weights_l.append(k[i : i + 1, :, :]) + kv_weights_l.append(v[i : i + 1, :, :]) + kv_weights = torch.cat(kv_weights_l) + + kv_weights = kv_weights.reshape([2 * hidden_size, hidden_size]) + + return kv_weights + + +@io.model_importer(CosmosVideo2WorldModel, "pt") +class PTCosmosVideo2WorldImporter(io.ModelConnector["PTCosmosVideo2WorldModel", CosmosVideo2WorldModel]): + def init(self) -> CosmosVideo2WorldModel: + return CosmosVideo2WorldModel(self.config, tokenizer=self.tokenizer) + + def apply(self, output_path: Path) -> Path: + pt_model_path = str(self) + cosmos_model_state_dict = torch.load(pt_model_path, map_location="cpu") + for k, v in cosmos_model_state_dict.items(): + # convert to float 32 (for cpu conversion) (Original model is bf16) + cosmos_model_state_dict[k] = v.float() + + # Small wrapper since nemo calls source.state_dict() , to get state dict + class WrapperCosmos: + def __init__(self, model_state_dict): + self.model_state_dict = model_state_dict + + def state_dict(self): + return self.model_state_dict + + source = WrapperCosmos(cosmos_model_state_dict) + target = self.init() + trainer = self.nemo_setup(target) + self.convert_state(source, target) + self.nemo_save(output_path, trainer) + + log.info(f"Converted PT Cosmos model to Nemo, model saved to {output_path}") + + teardown(trainer, target) + del trainer, target + + return output_path + + def convert_state(self, source, target): + mapping = { + "model.tok_embeddings.weight": "embedding.word_embeddings.weight", + "model.layers.*.attention.wo.weight": "decoder.layers.*.self_attention.linear_proj.weight", + "model.layers.*.attention.q_norm.weight": "decoder.layers.*.self_attention.q_layernorm.weight", + "model.layers.*.attention.k_norm.weight": "decoder.layers.*.self_attention.k_layernorm.weight", + "model.layers.*.attention_norm.weight": "decoder.layers.*.self_attention.linear_qkv.layer_norm_weight", + "model.layers.*.feed_forward.w2.weight": "decoder.layers.*.mlp.linear_fc2.weight", + "model.layers.*.ffn_norm.weight": "decoder.layers.*.mlp.linear_fc1.layer_norm_weight", + "model.layers.*.cross_attention.wo.weight": "decoder.layers.*.cross_attention.linear_proj.weight", + "model.layers.*.cross_attention.wq.weight": "decoder.layers.*.cross_attention.linear_q.weight", + "model.layers.*.cross_attention.q_norm.weight": "decoder.layers.*.cross_attention.q_layernorm.weight", + "model.layers.*.cross_attention.k_norm.weight": "decoder.layers.*.cross_attention.k_layernorm.weight", + "model.layers.*.cross_attention.weight": "decoder.layers.*.cross_attention.linear_qkv.layer_norm_weight", + "model.layers.*.cross_attention_norm.weight": "decoder.layers.*.pre_cross_attn_layernorm.weight", + "model.norm.weight": "decoder.final_layernorm.weight", + "model.output.weight": "output_layer.weight", + } + + return io.apply_transforms( + source, + target, + mapping=mapping, + transforms=[_import_qkv_cosmos_self_attention, _mlp_glu, _import_kv_cosmos_cross_attention], + ) + + @property + def tokenizer(self): + return None + + @property + def config(self): + if "5B" in str(self) or "5b" in str(self): + return CosmosConfigVideo2World5B() + elif "13B" in str(self) or "13b" in str(self): + return CosmosConfigVideo2World13B() + else: + raise ValueError("Unable to infer model size from checkpoint") diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/download_autoregressive_nemo.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/download_autoregressive_nemo.py new file mode 100644 index 00000000..6e97e0cd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/download_autoregressive_nemo.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +from huggingface_hub import snapshot_download + + +def download_autregressive_nemo(): + """ + Downloads all Cosmos Autoregressive NeMo assets to HF_HOME directory. + Make sure to set HF_HOME to your desired path before running this function. + """ + snapshot_download("nvidia/Cosmos-1.0-Guardrail") + snapshot_download("nvidia/Cosmos-1.0-Tokenizer-DV8x16x16") + snapshot_download("nvidia/Cosmos-1.0-Autoregressive-4B", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Autoregressive-12B", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Autoregressive-13B-Video2World", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Decoder-DV8x16x16ToCV8x8x8") + + +def main(): + # Check if HF_HOME is set + hf_home = os.environ.get("HF_HOME") + if not hf_home: + raise EnvironmentError( + "The HF_HOME environment variable is not set. " + "Please set it to your desired path before running this script." + ) + + # Download Cosmos Autoregressive NeMo checkpoints + download_autregressive_nemo() + + +if __name__ == "__main__": + main() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/README.md new file mode 100644 index 00000000..53ac0402 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/README.md @@ -0,0 +1,280 @@ +# Cosmos Autoregressive-based World Foundation Models: NeMo Framework User Guide + +Learn how to [run inference](#run-inference) with Cosmos Autoregressive-based World Foundation Models (WFMs) using the [NVIDIA NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html) for your custom Physical AI tasks by following this guide. + +## Model Support Matrix + +The NeMo Framework supports the following Cosmos Autoregressive (AR) models. Review the available models and their compute requirements for post-training and inference to determine the best model for your use case. + +| Model Name | Model Status | Compute Requirements for Inference | Multi-GPU Support | +|----------------------------------------------|------------------|------------------------------------------|---------| +| Cosmos-1.0-Autoregressive-4B | **Supported** | 1 NVIDIA GPU* | **Coming Soon** | +| Cosmos-1.0-Autoregressive-12B | **Supported** | 1 NVIDIA GPU* | **Coming Soon** | +| Cosmos-1.0-Autoregressive-5B-Video2World | **Supported** | 1 NVIDIA GPU* | **Coming Soon** | +| Cosmos-1.0-Autoregressive-13B-Video2World | **Supported** | 1 NVIDIA GPU* | **Coming Soon** | + +**\*** `H100-80GB` or `A100-80GB` GPUs are recommended. + +## Post-Training Inference Support Matrix + +Cosmos Autoregressive-based WFMs can be post-trained for a variety of Physical AI tasks. Review the following table for a list of available Physical AI post-training tasks: + +| Post-training Task | Inference Support Status | +|-------------------------|--------------------| +| General post-training | **Supported** | +| Instruction control | **Supported** | +| Action control | **Supported** | +| Camera control | **Coming Soon** | +| Multi-view generation | **Coming Soon** | +| Multi-view generation with vehicle trajectory control | **Coming Soon** | +| Changing the Video Tokenizer | **Supported** | + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the minimum compute required to run the model(s), as listed in the model support matrix. + - **Containerization Platform**: We recommend using Docker with NVIDIA Container Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Clone the Cosmos Repository + +```bash +git clone git@github.com:NVIDIA/Cosmos.git +``` + +### 3. Start the Container + +The [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos AR models. + +Run the following command to download and start the container: + + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidia/nemo:25.02.rc3 bash + ``` + +### 4. Download Checkpoints + +To help you get started, we've provided a [download script](../download_autoregressive_nemo.py) to get the Cosmos Autoregressive checkpoints from Hugging Face. These checkpoints are in the NeMo distributed checkpoint format required to run post-training and inference with NeMo Framework. + +1. Set the following environment variables: + + ```bash + # You must set HF_HOME before running this script. + export HF_TOKEN="" + export HF_HOME="" + ``` + +2. Run the following command to download the models: + + ```bash + cd /workspace/Cosmos + python cosmos1/models/autoregressive/nemo/download_autoregressive_nemo.py + ``` + +## Run Inference + +Running inference with Cosmos AR models lets you predict video frames and generate a new video that continues the scene from a given input video. + +In this guide, we'll use this [example inference script](./general.py) to tokenize the input video into a sequence of tokens, which serve as prompts for the model. The model then generates new tokens representing the next set of frames. Finally, the new tokens are decoded back into video format. Only the last 9 frames of the input video are used to generate the next 24 frames. + +### Run the Inference Script with Base Models + +#### 4B and 12B Models + +Complete the following steps to run inference on the 4B model. + +1. Set the following environment variables: + + ```bash + # Install required packages + pip install --no-cache-dir imageio[ffmpeg] pyav iopath better_profanity peft + + export HF_TOKEN="" + export HF_HOME="" + + # Path to the the mp4 file (In git-lfs) + export INPUT_DATA=cosmos1/models/autoregressive/assets/v1p0/input.mp4 + ``` + +2. Run the following command: + + ```bash + cd /workspace/Cosmos + git lfs pull $INPUT_DATA + + torchrun --nproc-per-node 1 cosmos1/models/autoregressive/nemo/inference/general.py \ + --input_image_or_video_path $INPUT_DATA \ + --video_save_name "Cosmos-1.0-Autoregressive-4B.mp4" \ + --ar_model_dir nvidia/Cosmos-1.0-Autoregressive-4B + ``` + +#### 5B and 13B Models + +Complete the following steps to run inference on the 5B model. + +1. Set the following environment variables: + + ```bash + # Install required packages + pip install --no-cache-dir imageio[ffmpeg] pyav iopath better_profanity peft git+https://github.com/NVlabs/Pytorch_Retinaface.git@b843f45 + + export HF_TOKEN= + export HF_HOME="" + + # Path to the the mp4 file (In git-lfs) + export INPUT_DATA=cosmos1/models/autoregressive/assets/v1p0/input.mp4 + ``` + +2. Run the following command: + + ```bash + cd /workspace/Cosmos + git lfs pull $INPUT_DATA + + torchrun --nproc-per-node=1 cosmos1/models/autoregressive/nemo/inference/video2world.py \ + --input_type video \ + --input_image_or_video_path 'cosmos1/models/autoregressive/assets/v1p0/input.mp4' \ + --prompt "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." \ + --ar_model_dir nvidia/Cosmos-1.0-Autoregressive-5B-Video2World + ``` + +### Run the Inference Script with Post-trained Models + +You must [create a post-trained model](../post_training/README.md) before completing this section. + +#### 4B and 12B Models + +Complete the following steps to generate a new output video using a post-trained Base model. + +1. Set the following environment variables: + + ```bash + pip install --no-cache-dir imageio[ffmpeg] pyav iopath better_profanity peft git+https://github.com/NVlabs/Pytorch_Retinaface.git@b843f45 + + export HF_TOKEN="" + export HF_HOME="" + + # Inference with post-trained model. + # NOTE: Dont use the checkpoint with -last suffix. + export NEMO_CHECKPOINT=./logs/default/checkpoints/epoch\=0-step\=9 + + # Path to the the mp4 file (In git-lfs) + export INPUT_DATA=cosmos1/models/autoregressive/assets/v1p0/input.mp4 + ``` + +2. Run the following command: + + ```bash + cd /workspace/Cosmos + git lfs pull $INPUT_DATA + export NUM_DEVICES=8 # change to your number of GPUs + + # change --ar_model_dir to a post-trained checkpoint under ./logs/default/checkpoints/ + torchrun --nproc-per-node $NUM_DEVICES cosmos1/models/autoregressive/nemo/inference/general.py \ + --input_image_or_video_path $INPUT_DATA \ + --video_save_name "Cosmos-1.0-Autoregressive-4B.mp4" \ + --ar_model_dir "$NEMO_CHECKPOINT" + ``` + +#### 5B and 13B Models + +Complete the following steps to generate a new output video using a post-trained Video2World model. + +1. Set the following environment variables: + + ```bash + pip install --no-cache-dir imageio[ffmpeg] pyav iopath better_profanity peft git+https://github.com/NVlabs/Pytorch_Retinaface.git@b843f45 + + export HF_TOKEN="" + export HF_HOME="" + + # Inference with post-trained model. + # NOTE: Dont use the checkpoint with -last suffix. + export NEMO_CHECKPOINT=./logs/default/checkpoints/epoch\=2-step\=9-last + + # Path to the the mp4 file (In git-lfs) + export INPUT_DATA=cosmos1/models/autoregressive/assets/v1p0/input.mp4 + + ``` + +2. Run the following command: + + ```bash + cd /workspace/Cosmos + git lfs pull $INPUT_DATA + + # change --ar_model_dir to a post-trained checkpoint under ./logs/default/checkpoints/ + torchrun --nproc-per-node=1 cosmos1/models/autoregressive/nemo/inference/video2world.py \ + --input_image_or_video_path $INPUT_DATA \ + --video_save_name "Cosmos-1.0-Autoregressive-5B-Video2World.mp4" \ + --prompt "A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." \ + --ar_model_dir "$NEMO_CHECKPOINT" + ``` + +#### Action Control Models + +First generate a post-trained [action control checkpoint](../post_training/action_control/README.md). + +1. Run the following command to download and start the container: + + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + -v :/root/.cache/huggingface \ + -v :/checkpoint/ \ + nvcr.io/nvidia/nemo:25.02.rc3 bash + ``` + +2. Set the following environment variables: + + ```bash + pip install -e Cosmos + export HF_HOME=/root/.cache/huggingface + export HF_TOKEN="" + ``` + +3. Run the inference script, choosing the desired frame to visualize. + + ```bash + python Cosmos/cosmos1/models/autoregressive/nemo/inference/action_control_infer.py /checkpoint \ + --output-dir Cosmos/ \ + --dataset-split val \ + --index 42 + ``` + +#### Example Output + +The following output is an example video generated from the post-trained model using [`general.py`](./general.py): + + + +Generated videos are saved at the location configured in the `--video_save_name` parameter. + +The input video used to generate this video can be found in `cosmos1/models/autoregressive/assets/v1p0/input.mp4`. + +> **Disclaimer**: +> The post-training example in this documentation is a demonstration of general post-training and not a guaranteed recipe for success. Post-training outcomes depend heavily on the quality and diversity of the dataset. To achieve good results, ensure your dataset is clean, well-structured, diverse, and properly labeled. Poorly prepared data can lead to issues like overfitting, bias, or poor performance. Carefully curate your dataset to reflect the desired use case for reliable results. + +### Configuration Options + +The following table details the parameters that can be modified for accelerated inference with NeMo. You can adjust these parameters to optimize performance based on your specific requirements + +| Parameter | Description | Default | +|--------------------------------|---------------------------------------------------------------------------------|---------| +| `--input_type` | The input type (image or video) | `video` | +| `--input_image_or_video_path` | Path to the input video to run inference | `cosmos1/models/autoregressive/assets/v1p0/input.mp4` | +| `--video_save_name` | Path to generated video | `./nemo_generated_video.mp4` | +| `--ar_model_dir` | Model name or path to model `nvidia/Cosmos-1.0-Autoregressive-4B` or `nvidia/Cosmos-1.0-Autoregressive-12B` | `nvidia/Cosmos-1.0-Autoregressive-4B` | +| `--encoder_path` | Path to encoder | `nvidia/Cosmos-1.0-Tokenizer-DV8x16x16` | +| `--decoder_path` | Path to the decoder | `nvidia/Cosmos-1.0-Tokenizer-DV8x16x1"` | +| `--guardrail_dir` | Path to guardrails | `nvidia/Cosmos-1.0-Guardrail` | +| `--top_p` | Top-p inference parameter | `0.9` | +| `--temperature` | Sampling temperature | `1` | +| `--disable_diffusion_decoder` | Disables running diffusion decoder on the generated result | `False` | diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/action_control_infer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/action_control_infer.py new file mode 100644 index 00000000..fd2a60b2 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/action_control_infer.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +from pathlib import Path + +import matplotlib.pyplot as plt +import torch +from cosmos1.models.autoregressive.nemo.inference.general import MockMCoreTokenizer +from cosmos1.models.autoregressive.nemo.inference.inference_controller import CosmosActionGenerationController +from cosmos1.models.autoregressive.nemo.post_training.action_control.action_control_dataset import ActionControlDataset +from cosmos1.models.autoregressive.nemo.post_training.action_control.prepare_dataset import Split, create_tokenizer +from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.engines.mcore_engine import MCoreEngine +from nemo import lightning as nl +from nemo.collections.llm.inference.base import _setup_trainer_and_restore_model +from nemo.lightning import io +from nemo.lightning.ckpt_utils import ckpt_to_context_subdir + + +LATENT_SHAPE = [1, 30, 40] # For the nvidia/Cosmos-1.0-Tokenizer-DV8x16x16 + + +BOV_TOKEN = 64000 +from einops import rearrange + + +def create_trainer(): + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + setup_optimizers=False, + store_optimizer_states=False, + ) + + trainer = nl.Trainer( + accelerator="gpu", + devices=1, + num_nodes=1, + strategy=strategy, + num_sanity_val_steps=0, + plugins=nl.MegatronMixedPrecision( + precision="bf16-mixed", + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + autocast_enabled=False, + grad_reduce_in_fp32=False, + ), + ) + + return trainer + + +def main( + checkpoint_directory: str, + temperature: float = 1, + top_p: float = 0.0, + dataset_split: Split = Split.val, + index: int = 0, + output_dir: str = "outputs/", +): + """Generate an example action control prediction from a post-trained checkpoint. + + Args: + checkpoint_directory: A path to a post-trained checkpoint directory. + temperature: The temperature to use for the prediction. + top_p: The top-p value to use for the prediction. + dataset_split: The dataset split to use for the prediction. + index: The index of the item to use for the prediction. + output_dir: The directory to save the output images to. + """ + + num_tokens_to_generate = math.prod(LATENT_SHAPE) + + # Create the mcore inference engine from the restored checkpoint to handle efficient caching of + # repeated auto-regressive token generation. + model: io.TrainerContext = io.load_context(path=ckpt_to_context_subdir(checkpoint_directory), subpath="model") + trainer = create_trainer() + _setup_trainer_and_restore_model(path=Path(checkpoint_directory), trainer=trainer, model=model) + inference_wrapped_model = model.get_inference_wrapper(torch.bfloat16, inference_batch_times_seqlen_threshold=1000) + action_generation_controller = CosmosActionGenerationController( + inference_wrapped_model=inference_wrapped_model, tokenizer=MockMCoreTokenizer() + ) + mcore_engine = MCoreEngine(text_generation_controller=action_generation_controller, max_batch_size=1) + + # Create the dataset and sample a single item from it. + dataset = ActionControlDataset(subfolder="autoregressive", split=dataset_split.value, shuffle=False) + sample = dataset[index] + input_frame = torch.cat([torch.tensor([BOV_TOKEN]), sample["current_frame"].flatten()]) + prompts: list[list[int]] = [input_frame.tolist()] + actions = sample["action"].unsqueeze(0).cuda() # shape([1, 7]), dtype=torch.bfloat16 + + # Generate the next frame by passing in the current frame and the action to take and + # subsequently generating prod(LATENT_SHAPE) tokens. Since we're only using a batch size of 1 + # here, we take the first item. + results = mcore_engine.generate( + prompts=prompts, + add_BOS=False, + encoder_prompts=actions, # type: ignore + common_inference_params=CommonInferenceParams( + num_tokens_to_generate=num_tokens_to_generate, + top_p=top_p, + temperature=temperature, + ), + )[0] + + # Reshape the generated tokens to the correct shape for an output frame. + predicted_frame = results.generated_tokens.reshape(1, 30, 40).detach().cpu() + + # Stack the input frame, ground truth next frame, and predicted next frame and decode them into RGB images together. + all_frames = torch.stack([sample["current_frame"], sample["next_frame"], predicted_frame]) + video_tokenizer = create_tokenizer() + decoded_image = video_tokenizer.decode(all_frames.cuda(), pixel_chunk_duration=None).detach() + decoded_image = (decoded_image + 1) * 127.5 + decoded_image = rearrange(decoded_image, "b c 1 h w -> b h w c").to(device="cpu", dtype=torch.uint8) + + # Create a single image with the three frames side by side. + text_labels = ["Input Frame", "Next Frame (Ground Truth)", "Next Frame (Predicted)"] + _, ax = plt.subplots(nrows=1, ncols=3, figsize=(9, 3)) + ax[0].imshow(decoded_image[0]) + ax[1].imshow(decoded_image[1]) + ax[2].imshow(decoded_image[2]) + for i, text in enumerate(text_labels): + ax[i].set_title(text) + output_path = Path(output_dir) / f"action_control_ar_{dataset_split.value}_{index}.png" + output_path.parent.mkdir(parents=True, exist_ok=True) + plt.savefig(output_path) + + +if __name__ == "__main__": + import typer + + typer.run(main) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/general.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/general.py new file mode 100644 index 00000000..5f65bd38 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/general.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import os +from argparse import ArgumentParser +from typing import List + +import imageio +import nemo.lightning as nl +import numpy as np +import torch +from cosmos1.models.autoregressive.nemo.utils import run_diffusion_decoder_model +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQJITTokenizer +from cosmos1.models.autoregressive.utils.inference import load_vision_input +from cosmos1.models.guardrail.common import presets as guardrail_presets +from cosmos1.utils import log +from einops import rearrange +from huggingface_hub import snapshot_download +from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.engines.mcore_engine import MCoreEngine +from megatron.core.inference.text_generation_controllers.simple_text_generation_controller import ( + SimpleTextGenerationController, +) +from nemo.collections.llm.inference.base import _setup_trainer_and_restore_model +from nemo.lightning import io +from nemo.lightning.ckpt_utils import ckpt_to_context_subdir + + +torch._C._jit_set_texpr_fuser_enabled(False) + +LATENT_SHAPE = [5, 40, 64] +NUM_INPUT_FRAMES_VIDEO = 9 + + +class MockMCoreTokenizer: + """ + A small dummy wrapper to pass into the text generation controller. + """ + + def __init__(self, vocab_size: int = 64000): + self.tokenizer = None + self.eod = -1 + self.vocab_size = vocab_size + + def detokenize(self, tokens: List[int], remove_special_tokens: bool = False): + return tokens + + def tokenize(self, prompt: List[int]): + return prompt + + +def main(args): + num_input_frames = 1 if args.input_type == "image" else NUM_INPUT_FRAMES_VIDEO + torch.distributed.init_process_group(backend="nccl") + TOKENIZER_COMPRESSION_FACTOR = list(map(int, args.tokenizer_compression_factor.split(","))) + NUM_CONTEXT_FRAMES = args.num_context_frames + + vision_input_dict = load_vision_input( + input_type=args.input_type, + batch_input_path=None, + input_image_or_video_path=args.input_image_or_video_path, + data_resolution=[args.height, args.width], + num_input_frames=num_input_frames, + num_total_frames=args.num_context_frames, + ) + + vision_input = list(vision_input_dict.values())[0].cuda() + vision_input = vision_input[:, :, :NUM_CONTEXT_FRAMES, :, :] + + T, H, W = LATENT_SHAPE + latent_context_t_size = 1 if args.input_type == "image" else 2 + num_tokens_to_generate = int(np.prod([T - latent_context_t_size, H, W])) + + # Encode and Tokenize + if args.encoder_path.startswith("nvidia/"): + args.encoder_path = os.path.join(snapshot_download(args.encoder_path), "encoder.jit") + if args.decoder_path.startswith("nvidia/"): + args.decoder_path = os.path.join(snapshot_download(args.decoder_path), "decoder.jit") + + video_tokenizer = DiscreteVideoFSQJITTokenizer( + enc_fp=args.encoder_path, + dec_fp=args.decoder_path, + name="discrete_video_fsq", + pixel_chunk_duration=NUM_CONTEXT_FRAMES, + latent_chunk_duration=T, + compression_ratio=TOKENIZER_COMPRESSION_FACTOR, + ).cuda() + + quantized_out, _ = video_tokenizer.encode(vision_input, pixel_chunk_duration=None) + indices = video_tokenizer.fsq_quantizer.codes_to_indices(quantized_out.permute(0, 2, 3, 4, 1)) + indices = rearrange(indices, "B T H W -> B (T H W)") + video_tokens = [indices[0][0:-num_tokens_to_generate].tolist()] + + # Load the nemo model + if args.ar_model_dir in ["nvidia/Cosmos-1.0-Autoregressive-4B", "nvidia/Cosmos-1.0-Autoregressive-12B"]: + args.ar_model_dir = os.path.join(snapshot_download(args.ar_model_dir, allow_patterns=["nemo/*"]), "nemo") + model: io.TrainerContext = io.load_context(path=ckpt_to_context_subdir(args.ar_model_dir), subpath="model") + + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=args.tensor_model_parallel_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + setup_optimizers=False, + store_optimizer_states=False, + ) + + trainer = nl.Trainer( + accelerator="gpu", + devices=torch.cuda.device_count(), + num_nodes=1, + strategy=strategy, + num_sanity_val_steps=0, + plugins=nl.MegatronMixedPrecision( + precision="bf16-mixed", + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + autocast_enabled=False, + grad_reduce_in_fp32=False, + ), + ) + + _setup_trainer_and_restore_model(path=args.ar_model_dir, trainer=trainer, model=model) + + inference_wrapped_model = model.get_inference_wrapper(torch.bfloat16, inference_batch_times_seqlen_threshold=1000) + + # Generate tokens + text_generation_controller = SimpleTextGenerationController( + inference_wrapped_model=inference_wrapped_model, tokenizer=MockMCoreTokenizer() + ) + + mcore_engine = MCoreEngine(text_generation_controller=text_generation_controller, max_batch_size=1) + + common_inference_params = CommonInferenceParams( + temperature=args.temperature, top_p=args.top_p, num_tokens_to_generate=num_tokens_to_generate + ) + + log.info(f"Running Inference to generate {num_tokens_to_generate} tokens. This will take some time. ") + results = mcore_engine.generate( + prompts=video_tokens, + add_BOS=False, + encoder_prompts=None, + common_inference_params=common_inference_params, + ) + + result = list(results)[0] + + prompt_tokens = torch.tensor(result.prompt_tokens).cuda() + prompt_tokens = torch.cat((prompt_tokens, result.generated_tokens)) + + indices_tensor = prompt_tokens.unsqueeze(dim=0) + indices_tensor = rearrange( + indices_tensor, + "B (T H W) -> B T H W", + T=LATENT_SHAPE[0], + H=LATENT_SHAPE[1], + W=LATENT_SHAPE[2], + ) + + if torch.cuda.current_device() == 0: + # Decode the generated tokens + log.info("Running diffusion model on the generated result") + video_decoded = video_tokenizer.decode(indices_tensor.cuda()) + out_video = (video_decoded * 0.5 + 0.5).clamp_(0, 1) + + if not args.disable_diffusion_decoder: + del model + del inference_wrapped_model + del video_tokenizer + model = None + inference_wrapped_model = None + video_tokenizer = None + gc.collect() + torch.cuda.empty_cache() + + out_video = run_diffusion_decoder_model( + indices_tensor_cur_batch=[indices_tensor.squeeze()], out_videos_cur_batch=out_video + ) + + out_video = out_video[0].detach().clone() + output_video = (out_video * 255).to(torch.uint8).permute(1, 2, 3, 0).cpu().numpy() + + if args.guardrail_dir: + log.info("Running guardrails on the generated video") + if args.guardrail_dir == "nvidia/Cosmos-1.0-Guardrail": + args.guardrail_dir = snapshot_download(args.guardrail_dir) + video_guardrail = guardrail_presets.create_video_guardrail_runner(checkpoint_dir=args.guardrail_dir) + output_video = guardrail_presets.run_video_guardrail(output_video, video_guardrail) + if output_video is None: + raise ValueError("Guardrail blocked world generation.") + + # Write the video to disk + imageio.mimsave( + args.video_save_name, + output_video, + fps=25, # We use a fps of 25 just for visualization. + ) + + log.info(f"Saved to {args.video_save_name}") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--width", required=False, default=1024, type=int, help="Width of the input videos") + parser.add_argument("--height", required=False, default=640, type=int, help="Height of the input videos") + parser.add_argument("--num_context_frames", required=False, default=33, type=int, help="Number of context frames") + parser.add_argument( + "--tokenizer_compression_factor", + required=False, + default="8,16,16", + type=str, + help="Tokenizer compression factor", + ) + parser.add_argument("--input_type", type=str, default="video", help="Type of input", choices=["image", "video"]) + parser.add_argument( + "--input_image_or_video_path", required=True, type=str, help="The path to the input video to run inference" + ) + parser.add_argument( + "--video_save_name", default="./nemo_generated_video.mp4", type=str, help="The path to generated video" + ) + parser.add_argument("--num_input_frames", default=9, type=int, help="The number of input frames") + parser.add_argument( + "--ar_model_dir", + default="nvidia/Cosmos-1.0-Autoregressive-4B", + type=str, + help="The path to the nemo autoregressive model", + ) + parser.add_argument( + "--encoder_path", default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", type=str, help="The path to encoder" + ) + parser.add_argument( + "--decoder_path", default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", type=str, help="The path to the decoder" + ) + parser.add_argument( + "--guardrail_dir", default="nvidia/Cosmos-1.0-Guardrail", type=str, help="The path to the guardrails" + ) + parser.add_argument("--top_p", default=0.8, type=float, help="The top_p inference parameter ") + parser.add_argument("--temperature", default=1, type=int, help="Sampling temperature") + parser.add_argument("--disable_diffusion_decoder", action="store_true", help="Disable diffusion decoder") + parser.add_argument( + "--tensor_model_parallel_size", default=torch.cuda.device_count(), type=int, help="The number of GPUs to use" + ) + + args = parser.parse_args() + + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/inference_controller.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/inference_controller.py new file mode 100644 index 00000000..abf4bbba --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/inference_controller.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any + +import torch +from cosmos1.models.autoregressive.modules.embedding import SinCosPosEmbAxisTE +from megatron.core import tensor_parallel +from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import GPTInferenceWrapper +from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import InferenceWrapperConfig +from megatron.core.inference.text_generation_controllers.simple_text_generation_controller import ( + SimpleTextGenerationController, +) +from megatron.core.models.gpt import GPTModel +from megatron.core.models.gpt.gpt_model import GPTModel as MCoreGPTModel + + +class CosmosInferenceWrapper(GPTInferenceWrapper): + def __init__( + self, + model: GPTModel, + inference_wrapper_config: InferenceWrapperConfig, + config, #: CosmosVideo2WorldConfig + ): + super().__init__(model, inference_wrapper_config) + self.config = config + + def prep_model_for_inference(self, prompts_tokens: torch.Tensor): + super().prep_model_for_inference(prompts_tokens=prompts_tokens) + self.abs_pos_emb = self._initialize_abs_pos_emb() + + def _initialize_abs_pos_emb(self): + pos_emb = SinCosPosEmbAxisTE( + dim=self.config.hidden_size, + latent_shape=self.config.latent_shape, + pad_to_multiple_of=self.config.pad_to_multiple_of, + ) + abs_pos_emb = pos_emb.forward(training_type=self.config.training_type) + abs_pos_emb = abs_pos_emb.transpose(0, 1).contiguous() + return abs_pos_emb + + def get_batch_for_context_window( + self, inference_input, context_start_position: int, context_end_position: int + ) -> dict[str, Any]: + data_at_step_idx = super().get_batch_for_context_window( + inference_input, context_start_position, context_end_position + ) + absposembed2use = self.abs_pos_emb[context_start_position:context_end_position, :, :] + data_at_step_idx["extra_positional_embeddings"] = absposembed2use + return data_at_step_idx + + def set_context_tokens(self, batch_context_tokens): + self.context_tokens = batch_context_tokens + + def forward_pass_without_pipeline_parallel(self, inference_input: dict[str, Any]) -> torch.Tensor: + tokens = inference_input["tokens"] + position_ids = inference_input["position_ids"] + attention_mask = inference_input["attention_mask"] + abs_pos_embed = inference_input["extra_positional_embeddings"] + + assert hasattr(self, "context_tokens"), ( + "Expected to have context tokens. Not present. Call set_context_tokens with the encoder embeddings" + ) + extra_block_kwargs = {"context": self.context_tokens, "extra_positional_embeddings": abs_pos_embed} + packed_seq_params = None + + logits = self.model( + tokens, + position_ids, + attention_mask, + inference_params=self.inference_params, + packed_seq_params=packed_seq_params, + extra_block_kwargs=extra_block_kwargs, + ) + logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) + self.inference_params.sequence_len_offset += tokens.size(1) + + return logits + + +class CosmosTextGenerationController(SimpleTextGenerationController): + def generate_all_output_tokens_static_batch(self, active_requests, active_streams=None): + batch_context_tokens = ( + list(map(lambda request: request.encoder_prompt, active_requests.values()))[0] + .to(torch.bfloat16) + .permute(1, 0, 2) + ) + self.inference_wrapped_model.set_context_tokens(batch_context_tokens) + active_requests = super().generate_all_output_tokens_static_batch(active_requests, active_streams) + return active_requests + + +class CosmosActionControlInferenceWrapper(CosmosInferenceWrapper): + def forward_pass_without_pipeline_parallel(self, inference_input: dict[str, Any]) -> torch.Tensor: + tokens = inference_input["tokens"] + position_ids = inference_input["position_ids"] + attention_mask = inference_input["attention_mask"] + abs_pos_embed = inference_input["extra_positional_embeddings"] + + assert hasattr(self, "context_tokens"), ( + "Expected to have context tokens. Not present. Call set_context_tokens with the encoder embeddings" + ) + mcore_model = self.model + while not isinstance(mcore_model, MCoreGPTModel) and hasattr(mcore_model, "module"): + mcore_model = mcore_model.module + action_mlp_out = mcore_model.action_mlp(self.context_tokens.cuda()) + context = action_mlp_out.unsqueeze(0).repeat(self.config.action_dim, 1, 1) + + extra_block_kwargs = {"extra_positional_embeddings": abs_pos_embed.cuda(), "context": context} + packed_seq_params = None + logits = self.model( + tokens.cuda(), + position_ids.cuda(), + attention_mask.cuda(), + inference_params=self.inference_params, + packed_seq_params=packed_seq_params, + extra_block_kwargs=extra_block_kwargs, + ) + + logits = tensor_parallel.gather_from_tensor_model_parallel_region(logits) + self.inference_params.sequence_len_offset += tokens.size(1) + + return logits + + +class CosmosActionGenerationController(SimpleTextGenerationController): + def generate_all_output_tokens_static_batch(self, active_requests, active_streams=None): + batch_actions = list(map(lambda request: request.encoder_prompt, active_requests.values()))[0].to( + torch.bfloat16 + ) + self.inference_wrapped_model.set_context_tokens(batch_actions) + active_requests = super().generate_all_output_tokens_static_batch(active_requests, active_streams) + return active_requests diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/video2world.py new file mode 100644 index 00000000..5bdd04bd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/inference/video2world.py @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import os +from argparse import ArgumentParser + +import imageio +import nemo.lightning as nl +import numpy as np +import torch +from cosmos1.models.autoregressive.nemo.inference.general import MockMCoreTokenizer +from cosmos1.models.autoregressive.nemo.inference.inference_controller import CosmosTextGenerationController +from cosmos1.models.autoregressive.nemo.utils import run_diffusion_decoder_model +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQJITTokenizer +from cosmos1.models.autoregressive.utils.inference import load_vision_input +from cosmos1.models.common.t5_text_encoder import CosmosT5TextEncoder +from cosmos1.models.guardrail.common import presets as guardrail_presets +from cosmos1.utils import log +from einops import rearrange +from huggingface_hub import snapshot_download +from megatron.core.inference.common_inference_params import CommonInferenceParams +from megatron.core.inference.engines.mcore_engine import MCoreEngine +from nemo.collections.llm.inference.base import _setup_trainer_and_restore_model +from nemo.lightning import io +from nemo.lightning.ckpt_utils import ckpt_to_context_subdir + + +torch._C._jit_set_texpr_fuser_enabled(False) + +NUM_INPUT_FRAMES_VIDEO = 9 +LATENT_SHAPE = [5, 40, 64] + +BOV_TOKEN = 64000 +PAD_ID = 64002 +PAD_TO_MULTIPLE_OF = 64 + + +def get_text_prompt_embeddings(args): + text_encoder = CosmosT5TextEncoder() + prompt_embedding, prompt_mask = text_encoder.encode_prompts([args.prompt]) + del text_encoder + gc.collect() + torch.cuda.empty_cache() + return prompt_embedding + + +def main(args): + num_input_frames = 1 if args.input_type == "image" else NUM_INPUT_FRAMES_VIDEO + data_resolution = [args.height, args.width] + vision_input_dict = load_vision_input( + input_type=args.input_type, + batch_input_path=None, + input_image_or_video_path=args.input_image_or_video_path, + data_resolution=data_resolution, + num_input_frames=num_input_frames, + num_total_frames=args.num_context_frames, + ) + + vision_input = list(vision_input_dict.values())[0].cuda() + + T, H, W = LATENT_SHAPE + latent_context_t_size = 1 if args.input_type == "image" else 2 + num_tokens_to_generate = int(np.prod([T - latent_context_t_size, H, W])) + + # Encode and Tokenize + if args.encoder_path.startswith("nvidia/"): + args.encoder_path = os.path.join(snapshot_download(args.encoder_path), "encoder.jit") + if args.decoder_path.startswith("nvidia/"): + args.decoder_path = os.path.join(snapshot_download(args.decoder_path), "decoder.jit") + num_context_frames = args.num_context_frames + tokenizer_compression_factor = list(map(int, args.tokenizer_compression_factor.split(","))) + + video_tokenizer = DiscreteVideoFSQJITTokenizer( + enc_fp=args.encoder_path, + dec_fp=args.decoder_path, + name="discrete_video_fsq", + pixel_chunk_duration=num_context_frames, + latent_chunk_duration=T, + compression_ratio=tokenizer_compression_factor, + ).cuda() + + quantized_out, _ = video_tokenizer.encode(vision_input, pixel_chunk_duration=None) + indices = video_tokenizer.fsq_quantizer.codes_to_indices(quantized_out.permute(0, 2, 3, 4, 1)) + indices = rearrange(indices, "B T H W -> B (T H W)") + video_tokens = [[BOV_TOKEN] + indices[0][0:-num_tokens_to_generate].tolist()] + + # Load the nemo model + if args.ar_model_dir in [ + "nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", + "nvidia/Cosmos-1.0-Autoregressive-13B-Video2World", + ]: + args.ar_model_dir = os.path.join(snapshot_download(args.ar_model_dir, allow_patterns=["nemo/*"]), "nemo") + model: io.TrainerContext = io.load_context(path=ckpt_to_context_subdir(args.ar_model_dir), subpath="model") + + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + setup_optimizers=False, + store_optimizer_states=False, + ) + + trainer = nl.Trainer( + accelerator="gpu", + devices=1, + num_nodes=1, + strategy=strategy, + num_sanity_val_steps=0, + plugins=nl.MegatronMixedPrecision( + precision="bf16-mixed", + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + autocast_enabled=False, + grad_reduce_in_fp32=False, + ), + ) + _setup_trainer_and_restore_model(path=args.ar_model_dir, trainer=trainer, model=model) + + inference_wrapped_model = model.get_inference_wrapper(torch.bfloat16, inference_batch_times_seqlen_threshold=1000) + + text_generation_controller = CosmosTextGenerationController( + inference_wrapped_model=inference_wrapped_model, tokenizer=MockMCoreTokenizer(64064) + ) + + mcore_engine = MCoreEngine(text_generation_controller=text_generation_controller, max_batch_size=1) + + common_inference_params = CommonInferenceParams( + temperature=args.temperature, top_p=args.top_p, num_tokens_to_generate=num_tokens_to_generate + ) + + log.info(f"Running Inference to generate {num_tokens_to_generate} tokens. This will take some time. ") + + prompt_embedding = get_text_prompt_embeddings(args) + results = mcore_engine.generate( + prompts=video_tokens, + encoder_prompts=[prompt_embedding], + add_BOS=False, + common_inference_params=common_inference_params, + ) + + result = list(results)[0] + + prompt_tokens_with_bov = torch.tensor(result.prompt_tokens).cuda() + all_tokens = torch.cat((prompt_tokens_with_bov, result.generated_tokens)) + indices_tensor = all_tokens.unsqueeze(dim=0)[:, 1:] # To remove BOV Token + indices_tensor = rearrange( + indices_tensor, + "B (T H W) -> B T H W", + T=LATENT_SHAPE[0], + H=LATENT_SHAPE[1], + W=LATENT_SHAPE[2], + ) + + if torch.cuda.current_device() == 0: + # Decode the generated tokens + log.info("Running diffusion model on the generated result") + video_decoded = video_tokenizer.decode(indices_tensor.cuda()) + out_video = (video_decoded * 0.5 + 0.5).clamp_(0, 1) + + if not args.disable_diffusion_decoder: + del model + del inference_wrapped_model + del video_tokenizer + model = None + inference_wrapped_model = None + video_tokenizer = None + gc.collect() + torch.cuda.empty_cache() + + out_video = run_diffusion_decoder_model( + indices_tensor_cur_batch=[indices_tensor.squeeze()], out_videos_cur_batch=out_video + ) + + out_video = out_video[0].detach().clone() + output_video = (out_video * 255).to(torch.uint8).permute(1, 2, 3, 0).cpu().numpy() + + if args.guardrail_dir: + log.info("Running guardrails on the generated video") + if args.guardrail_dir == "nvidia/Cosmos-1.0-Guardrail": + args.guardrail_dir = snapshot_download(args.guardrail_dir) + video_guardrail = guardrail_presets.create_video_guardrail_runner(checkpoint_dir=args.guardrail_dir) + output_video = guardrail_presets.run_video_guardrail(output_video, video_guardrail) + if output_video is None: + raise ValueError("Guardrail blocked world generation.") + + # Write the video to disk + imageio.mimsave( + args.video_save_name, + output_video, + fps=25, # We use a fps of 25 just for visualization. + ) + + log.info(f"Saved to {args.video_save_name}") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--input_type", type=str, default="video", help="Type of input", choices=["image", "video"]) + parser.add_argument("--num_context_frames", type=int, default=33, help="Number of context frames") + parser.add_argument("--width", type=int, default=1024, help="Width of the video") + parser.add_argument( + "--tokenizer_compression_factor", type=str, default="8,16,16", help="Tokenizer compression factor" + ) + parser.add_argument("--height", type=int, default=640, help="Height of the video") + parser.add_argument( + "--input_image_or_video_path", required=True, type=str, help="The path to the input video to run inference" + ) + parser.add_argument("--prompt", type=str, help="Text prompt for generating a single video") + parser.add_argument( + "--video_save_name", default="./nemo_generated_video.mp4", type=str, help="The path to generated video" + ) + parser.add_argument( + "--ar_model_dir", + default="nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", + type=str, + help="The path to the nemo autoregressive model", + ) + parser.add_argument( + "--encoder_path", default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", type=str, help="The path to encoder" + ) + parser.add_argument( + "--decoder_path", default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", type=str, help="The path to the decoder" + ) + parser.add_argument( + "--guardrail_dir", default="nvidia/Cosmos-1.0-Guardrail", type=str, help="The path to the guardrails" + ) + parser.add_argument("--top_p", default=0.8, type=float, help="The top_p inference parameter ") + parser.add_argument("--temperature", default=1, type=int, help="Sampling temperature") + parser.add_argument("--disable_diffusion_decoder", action="store_true", help="Disable diffusion decoder") + + args = parser.parse_args() + + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/README.md new file mode 100644 index 00000000..a3a540b7 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/README.md @@ -0,0 +1,274 @@ +# Cosmos Autoregressive-based World Foundation Models: NeMo Framework User Guide + +Learn how to [post-train](#post-train) Cosmos Autoregressive-based World Foundation Models (WFMs) using the [NVIDIA NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html) for your custom Physical AI tasks by following this guide. + +## Model Support Matrix + +The NeMo Framework supports the following Cosmos Autoregressive (AR) models. Review the available models and their compute requirements for post-training and inference to determine the best model for your use case. + +| Model Name | Model Status | Compute Requirements for Post-Training | +|-------------------------|----------------------------|-------------------------------------------| +| Cosmos-1.0-Autoregressive-4B | **Supported** | 2 NVIDIA GPUs* | +| Cosmos-1.0-Autoregressive-12B | **Supported** | 8 NVIDIA GPUs* | +| Cosmos-1.0-Autoregressive-5B-Video2World | **Supported** | 2 NVIDIA GPUs* | +| Cosmos-1.0-Autoregressive-13B-Video2World | **Supported** | 8 NVIDIA GPUs* | + +**\*** `H100-80GB` or `A100-80GB` GPUs are recommended. + +## Post-Training Support Matrix + +Cosmos Autoregressive-based WFMs can be post-trained for a variety of Physical AI tasks. Review the following table for a list of available Physical AI post-training tasks: + +| Post-training Task | Support Status | +|-------------------------|--------------------| +| General post-training | **Supported** | +| Instruction control | **Supported** | +| Action control | **Supported** | +| Camera control | **Coming Soon** | +| Multi-view generation | **Coming Soon** | +| Multi-view generation with vehicle trajectory control | **Coming Soon** | +| Changing the Video Tokenizer | **Supported** | + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the minimum compute required to run the model(s), as listed in the model support matrix. + - **Containerization Platform**: We recommend using Docker with NVIDIA Container Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Clone the Cosmos Repository + +```bash +git clone git@github.com:NVIDIA/Cosmos.git +``` + +### 3. Start the Container + +The [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos AR models. + +Run the following command to download and start the container: + + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidia/nemo:25.02.rc3 bash + ``` + +### 4. Download Checkpoints + +To help you get started, we've provided a [download script](../download_autoregressive_nemo.py) to get the Cosmos Autoregressive checkpoints from Hugging Face. These checkpoints are in the NeMo distributed checkpoint format required to run post-training and inference with NeMo Framework. + +1. Set the following environment variables: + + ```bash + # You must set HF_HOME before running this script. + export HF_TOKEN="" + export HF_HOME="" + ``` + +2. Run the following command to download the models: + + ```bash + cd /workspace/Cosmos + python cosmos1/models/autoregressive/nemo/download_autoregressive_nemo.py + ``` + +## Post-train + +Post-training a Cosmos Autoregressive-based WFM enables you to train the model to generate videos using frame predictions that are more specific to your Physical AI use case. + +For example, if you want to generate action sequences for a specific robot, you can post-train the model to generate videos that are more aligned with typical actions/outcomes for that robot. + +There are 3 steps to post-training: preparing a dataset, preprocessing the data, and post-training the model. + +### 1. Prepare a Dataset + +The first step is to prepare a dataset. Post-training a Cosmos-1.0-Autoregressive model enables you to get better video-frame predictions for your specific use case. + +You must provide a folder containing a collection of videos in **MP4 format**, preferably 720p. In this guide, we'll use the sample videos located in the `cosmos1/models/autoregressive/assets/v1p0/batch_inputs` directory. + +### 2. Preprocess Data + +#### 4B and 12B Models +The second step is to preprocess the data to create an [indexed dataset](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core/datasets). + +The `IndexedDataset` class is the lowest-level data interface in Megatron Core and creates a `.bin` and `.idx` file. + +Before proceeding, ensure all videos are in **RGB format**. Complete the following steps to preprocess the data. + +1. Set the following environment variables: + + ```bash + pip install --no-cache-dir imageio[ffmpeg] pyav iopath + + export HF_TOKEN="" + export HF_HOME="" + + # Path to Raw mp4 videos. + export RAW_DATA="cosmos1/models/autoregressive/assets/v1p0/batch_inputs" + + # Path to Processed Dataset. + export OUTPUT_PREFIX="./indexed_videos" + ``` + +2. Run the following command to preprocess the data: + + ```bash + cd /workspace/Cosmos + git lfs pull --include=$RAW_DATA + + python cosmos1/models/autoregressive/nemo/post_training/prepare_dataset.py \ + --input_videos_dir $RAW_DATA \ + --output_prefix $OUTPUT_PREFIX + ``` + +Executing the [data preprocessing script](./prepare_dataset.py) for the base model generates the following files for each video: + +- **`[i].idx` File**: This file contains metadata at the dataset level: + - **Index Header**: Ensures backward compatibility. + - **Index Version**: Maintains backward compatibility. + - **Data Type Code**: Numeric code indicating the data type used in the data file. + - **Sequence Count**: Total number of sequences in the dataset. + - **Document Count**: Total number of documents in the dataset. + +- **`[i].bin` File**: This file includes metadata at the document and sequence levels: + - **Elements per Sequence**: Number of elements in each sequence. + - **Byte Offset per Sequence**: Pointer indicating the start of each sequence. + - **Sequence Index Range**: Consecutive index range `[...)` for each document. + +#### 5B and 13B Models +The second step is to preprocess the data to pre compute the text and video embeddings for finetuning.. + +Before proceeding, ensure all videos are in **RGB format**. Complete the following steps to preprocess the data. + +1. Set the following environment variables: + + ```bash + pip install --no-cache-dir imageio[ffmpeg] pyav iopath + + export HF_TOKEN="" + export HF_HOME="" + + # Path to Raw mp4 videos. + export RAW_DATA="cosmos1/models/autoregressive/assets/v1p0/batch_inputs" + + # Path to Processed Dataset. + export OUTPUT_PREFIX="./indexed_videos" + ``` + +2. Run the following command to preprocess the data: + + ```bash + cd /workspace/Cosmos + git lfs pull --include=$RAW_DATA + + python3 cosmos1/models/autoregressive/nemo/post_training/video2world_prepare_dataset.py \ + --input_jsonl $RAW_DATA/video2world.jsonl \ + --output_dir $OUTPUT_PREFIX + ``` + +Executing the [data preprocessing script](./video2world_prepare_dataset.py) for the base model generates + +Executing the [data preprocessing script](./prepare_dataset.py) for the base model generates the following files for each video: + +- **`[i].pt` File**: This file contains video tokens or prompt embeddings: + - It has a format `__.pt` + +- **`[i]metadata.json` File**: This file includes metadata: + - It tells you the number of train test and validation samples + +### 3. Post-train the Model + +The third step is to post-train the model. This step uses NeMo Framework's data and model parallelism capabilities to train the model on the post-training samples. This is accomplished by utilizing Tensor Parallelism. + +- **Tensor Parallelism**: Spreads the parameter tensor of individual layers across GPUs. + +#### Run the Post-training Script + +##### 4B AND 12B Models + +Complete the following steps to post-train the Cosmos-1.0-Autoregressive-4B model. + +1. Set the following environment variables: + + ```bash + export HF_TOKEN="" + export HF_HOME="" + + # Number of GPU devices available for post-training. At least 2 for 4B and 8 for 12B. + export NUM_DEVICES=2 + + # Optionally, you can monitor training progress with Weights and Biases (wandb). + export WANDB_API_KEY="" + export WANDB_PROJECT_NAME="cosmos-autoregressive-nemo-finetuning" + export WANDB_RUN_ID="cosmos_autoregressive_4b_finetune" + ``` + +2. Run the following command for Cosmos-1.0-Autoregressive-4B post-training: + + ```bash + torchrun --nproc-per-node $NUM_DEVICES cosmos1/models/autoregressive/nemo/post_training/general.py \ + --data_path $OUTPUT_PREFIX \ + --split_string 4,1,1 \ + --log_dir ./logs \ + --max_steps 10 --save_every_n_steps 5 \ + --tensor_model_parallel_size $NUM_DEVICES \ + --model_path nvidia/Cosmos-1.0-Autoregressive-4B + ``` + +3. You can now run inference with your post-trained model using the instructions [here](../inference/README.md#run-the-inference-script-with-post-trained-model). + +##### 5B and 13B Models + +Complete the following steps to post-train the Cosmos-1.0-Autoregressive-5B model. + +1. Set the following environment variables: + + ```bash + export HF_TOKEN="" + export HF_HOME="" + + # Number of GPU devices available for post-training. At least 4 for 5B and 8 for 13B. + export NUM_DEVICES=4 + + # Optionally, you can monitor training progress with Weights and Biases (wandb). + export WANDB_API_KEY="" + export WANDB_PROJECT_NAME="cosmos-autoregressive-nemo-finetuning" + export WANDB_RUN_ID="cosmos_autoregressive_5b_finetune" + ``` + +2. Run the following command for Cosmos-1.0-Autoregressive-5B-Video2World post-training: + + ```bash + torchrun --nproc-per-node $NUM_DEVICES \ + cosmos1/models/autoregressive/nemo/post_training/video2world_finetuning.py \ + --data_path $OUTPUT_PREFIX \ + --log_dir ./logs \ + --max_steps 10 --save_every_n_steps 5 \ + --tensor_model_parallel_size $NUM_DEVICES \ + --model_path nvidia/Cosmos-1.0-Autoregressive-5B-Video2World + ``` + +3. You can now run inference with your post-trained model using the instructions [here](../inference/README.md#run-the-inference-script-with-post-trained-model). + +#### Configuration Options + +Before getting started, review the following parameters that made available to the script. You can adjust these parameters to optimize performance based on your specific requirements. + +| Parameter | Description | Default | +|---|---|---| +| `--data_path` | Specifies the location of your preprocessed dataset. Ensure this path points to the directory containing your `.bin` and `.idx` files. | `/path/to/data` | +| `--model_path` | Specifies the directory to the cosmos model to run post-training on. | `nvidia/Cosmos-1.0-Autoregressive-4B` | +| `--index_mapping_dir` | Specifies the directory to store the indexed dataset. | `./index_mapping` | +| `--log_dir` | Specifies the directory to store the logs and checkpoints. | `./log_dir` | +| `--split_string` | Specifies the data split ratios for training, validation, and testing. (Only valid for Base Model (4B and 12B)) | `4,1,1` | +| `--tensor_model_parallel_size` | Controls the number of GPUs used for model parallelism. Increase this number to scale up, ensuring your hardware can support the additional load. | `2` | +| `--max_steps` | Defines the total number of training steps. Adjust based on training duration and storage capacity. | `100` | +| `--save_every_n_steps` | Defines how often checkpoints are saved. Adjust based on training duration and storage capacity. | `10` | +| `--global_batch_size` | Tweaks to optimize memory usage and training speed. Larger batch sizes may improve convergence but require more memory. | `2` | +| `--micro_batch_size` | Tweaks to optimize memory usage and training speed. Larger batch sizes may improve convergence but require more memory. | `1` | +| `--lr` | Sets the learning rate. A common starting point is `5e-5`, but this can be adjusted based on model performance and convergence behavior. | `5e-5` | +| `--max_epochs` | The maximum number of epochs to run during post-training | `10` | diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/README.md new file mode 100644 index 00000000..64a68fd3 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/README.md @@ -0,0 +1,66 @@ +# Action-Control Autoregressive Post-Training + +1. Run the following command to download and start the container: + + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + -v :/root/.cache/huggingface \ + nvcr.io/nvidia/nemo:25.02.rc1 bash + ``` + +2. Set the following environment variables: + + ``` + pip install --no-cache-dir imageio[ffmpeg] pyav iopath + export HF_HOME=/root/.cache/huggingface + export HF_TOKEN= + export PYTHONPATH=/workspace/Cosmos:$PYTHONPATH + ``` + +3. Run the following command to download the models: + + ```bash + python Cosmos/cosmos1/models/autoregressive/nemo/download_autoregressive_nemo.py + ``` +> [!NOTE] +> The full bridge dataset is approximately 30Gb, and the full download can take approximately 2 +> hours depending on your connection speed. To use a smaller, 10Mb sanity-sized dataset bundled with +> the Cosmos repository, pass `--dataset-dir Cosmos/cosmos1/models/autoregressive/assets/bridge` to +> the commands below. + +4. Then run the `prepare_dataset` script for the train, test, and val splits. The bridge dataset +will be downloaded automatically the first time you run it, and stored in +`HF_HOME/assets/cosmos/action-control/datasets/`. By default, the tokenized videos will be stored in +`HF_HOME/assets/cosmos/action-control/autoregressive/bridge/`. + + ``` + python Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py \ + --dataset-split train --batch-size 50 --num-workers 16 + + python Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py \ + --dataset-split val --batch-size 50 --num-workers 16 + + python Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py \ + --dataset-split test --batch-size 50 --num-workers 16 + ``` + +5. Run post-training. If no additional paths are provided, the pre-processed data will be loaded + from `HF_HOME`. + + ``` + export NUM_DEVICES=8 + # Data-parallel batch size per TP group. On 8xA100, we use 16 for 5B and 4 for 13B. + export MICRO_BATCH_SIZE=16 + export WANDB_API_KEY="" + export WANDB_PROJECT_NAME="cosmos-autoregressive-nemo-finetuning" + export WANDB_RUN_ID="cosmos_autoregressive_5b_action_control" + + torchrun --nproc-per-node $NUM_DEVICES \ + Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_finetuning.py \ + --log_dir ./logs \ + --max_steps 10 --save_every_n_steps 5 \ + --tensor_model_parallel_size $NUM_DEVICES \ + --micro_batch_size $MICRO_BATCH_SIZE \ + --model_path nvidia/Cosmos-1.0-Autoregressive-5B-Video2World + ``` diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_ar_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_ar_dataset.py new file mode 100644 index 00000000..165c3e79 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_ar_dataset.py @@ -0,0 +1,186 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from typing import Literal, TypedDict + +import torch +from nemo.collections.llm.gpt.data.mock import MockDataModule +from nemo.core.classes.dataset import Dataset + +from cosmos1.models.autoregressive.nemo.cosmos import CosmosConfig +from cosmos1.models.autoregressive.nemo.post_training.action_control.action_control_dataset import ActionControlDataset +from cosmos1.models.autoregressive.nemo.post_training.video2world_dataset import get_abs_pos_embed + + +BOV_TOKEN = 64000 +PAD_ID = 64002 + + +class ActionControlARBatch(TypedDict): + """A processed batch of data to include additonal keys needed by the Video2World model.""" + + tokens: torch.Tensor # shape: (B, T) + position_ids: torch.Tensor # shape: (B, T) + attention_mask: torch.Tensor # shape: (B, T, T) + labels: torch.Tensor # shape: (B, T) + abs_pos_embed: torch.Tensor # shape: (B, T, H, W) + action: torch.Tensor # shape: (B, 7) # Converted to context via MLP + loss_mask: torch.Tensor # shape: (B, T) + + +def ar_collate_fn(batch: list[ActionControlARBatch]) -> ActionControlARBatch: + op = torch.utils.data.dataloader.default_collate(batch) + op["attention_mask"] = op["attention_mask"][0, :, :, :].unsqueeze(dim=0) + op["abs_pos_embed"] = op["abs_pos_embed"][0, :, :, :] + return op + + +class ActionControlARDataset(Dataset): + """The action-control autoregressive post-training dataset.""" + + def __init__( + self, + model_config: CosmosConfig, + data_path: str | os.PathLike | None = None, + subfolder: str | None = None, + split: Literal["train", "val", "test"] = "train", + ): + self._base_ds = ActionControlDataset(data_path=data_path, subfolder=subfolder, split=split) + self._collate_fn = ar_collate_fn + self._abs_pos_embed = get_abs_pos_embed(model_config, training_type="text_to_video") + + def __len__(self) -> int: + return len(self._base_ds) + + def __getitem__(self, i: int) -> ActionControlARBatch: + row = self._base_ds[i] + + # Stack the starting frame and next frame into a single tensor, prepended with the BOV token + # and pad it to be divisible by 128. + tokens = torch.cat( + ( + torch.Tensor([BOV_TOKEN]), + row["current_frame"].view(-1), # shape: (1200,) for DV8x16x16 + row["next_frame"].view(-1), # shape: (1200,) for DV8x16x16 + torch.Tensor([PAD_ID] * 32), + ), + dim=0, + ).to(torch.int64) + + seq_len = tokens.numel() - 1 + frame_len = row["current_frame"].numel() + assert seq_len % 128 == 0, "sequence length should be divisible by 128" + assert seq_len == 2_432, f"sequence length should be 2_432, but is actually {seq_len}" + + attention_mask = ( + torch.tril(torch.ones((seq_len, seq_len), device=row["current_frame"].device)).unsqueeze(0).to(torch.bool) + ) + + loss_mask = torch.zeros(seq_len) + loss_mask[frame_len : frame_len * 2] = 1 # Only predict the next frame in an autoregressive manner. + + # Like the video2world model, we offset the tokens in the labels by 1, so that the model + # predicts the next token from the current tokens in an autoregressive manner. + return { + "tokens": tokens[:-1], + "position_ids": torch.arange(0, seq_len, device=row["current_frame"].device), + "attention_mask": attention_mask, + "labels": tokens[1:], + "abs_pos_embed": self._abs_pos_embed, + "action": row["action"].to(torch.bfloat16), + "loss_mask": loss_mask, + } + + +class ActionControlDataModule(MockDataModule): + """The action-control autoregressive post-training dataloader.""" + + def __init__( + self, + *args, + model_config: CosmosConfig, + data_path: str | os.PathLike | None = None, + subfolder: str | None = None, + **kwargs, + ): + """Initialize the action-control autoregressive post-training dataloader. + + Args: + model_config: The model configuration to use for calculating the absolute position embedding. + data_path: The path to the data. If not provided, this will assume the data is stored in the + default location in the huggingface cache. + subfolder: The subfolder to use in HF_HOME/assets/cosmos/action-control. Should not be provided + if data_path is provided. + """ + super().__init__(*args, **kwargs) + self.data_path = data_path + self.subfolder = subfolder + self.model_config = model_config + + def setup(self, stage: str | None = None): + """Setup the action-control autoregressive post-training dataloader.""" + self._train_ds = ActionControlARDataset( + data_path=self.data_path, + subfolder=self.subfolder, + split="train", + model_config=self.model_config, + ) + self._validation_ds = ActionControlARDataset( + data_path=self.data_path, + subfolder=self.subfolder, + split="test", + model_config=self.model_config, + ) + self._test_ds = ActionControlARDataset( + data_path=self.data_path, + subfolder=self.subfolder, + split="val", + model_config=self.model_config, + ) + + +if __name__ == "__main__": + import typer + + from cosmos1.models.autoregressive.nemo.cosmos_action_control import CosmosConfigActionControl5B + + def print_shapes(subfolder: str = "autoregressive"): + example_dataset = ActionControlDataset(split="val", subfolder=subfolder) + + print(f"{len(example_dataset) = }") + for key, val in example_dataset[0].items(): + print(f"{key = }") + print(f"{val.shape = }") + + # For the DV8x16x16 tokenizer, the tokenized frames are 30x40. + # For the CV8x8x8 tokenizer, the tokenized frames are 60x80. + data_module = ActionControlDataModule(model_config=CosmosConfigActionControl5B(), subfolder=subfolder) + data_module.setup() + train_dataloader = data_module.train_dataloader() + + # The dataloader here isn't being wrapped with the MegatronDataSampler, so the batch size hasn't + # been set to anything other than 1. We can't do that outside a megatron context, so this just + # demonstrates that iterating over the dataloader works. + print(f"{len(train_dataloader) = }") + for batch in train_dataloader: + for key, val in batch.items(): + print(f"{key = }") + print(f"{val.shape = }") + break + + typer.run(print_shapes) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_dataset.py new file mode 100644 index 00000000..3a78919f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_dataset.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path +from typing import Literal, TypedDict + +import torch +from torch.utils.data import Dataset + +from cosmos1.models.autoregressive.nemo.post_training.action_control.prepare_dataset import get_default_output_prefix + + +class ActionControlRow(TypedDict): + """A single item loaded via the action-control post-training dataset.""" + + current_frame: torch.Tensor # shape: (1, H, W) + next_frame: torch.Tensor # shape: (1, H, W) + action: torch.Tensor # shape: (7,) + + +class ActionControlDataset(Dataset): + """The action-control autoregressive post-training dataset.""" + + def __init__( + self, + data_path: str | os.PathLike | None = None, + subfolder: str | None = None, + split: Literal["train", "val", "test"] = "train", + seed: int = 42, + shuffle: bool = True, + ): + """Initialize the action-control autoregressive post-training dataset. + + Args: + data_path: The path to the data. If not provided, this will assume the data is stored in the + default location in the huggingface cache. + subfolder: The subfolder to use in HF_HOME/assets/cosmos/action-control. Should not be provided + if data_path is provided. + split: The split to use. + """ + + if data_path is None: + self.data_path = get_default_output_prefix(split, subfolder) + else: + assert subfolder is None, "subfolder should not be provided if data_path is provided" + self.data_path = Path(data_path) + + self.tokenized_frames = torch.load(self.data_path / "tokenized-frames.pt", mmap=True) + self.actions = torch.load(self.data_path / "actions.pt") + + # Here we use the NaN mask created in prepare_dataset.py to filter out the final frames + # from each trajectory, as they can't be used as an input action. + self.valid_actions_indices = torch.where(~torch.isnan(self.actions).all(dim=1))[0] + + # Shuffle the training data. + if shuffle: + g = torch.Generator() + g.manual_seed(seed) + self.valid_actions_indices = self.valid_actions_indices[ + torch.randperm(len(self.valid_actions_indices), generator=g) + ] + + def __len__(self) -> int: + """The number of valid actions in the dataset. + + Since the last frame from each trajectory can't be used as an input action, this is less + than the total number of frames. + """ + return len(self.valid_actions_indices) + + def __getitem__(self, i: int) -> ActionControlRow: + """Get the i-th action-control batch from the dataset. + + Args: + i: The index of the batch to get. + + Returns: + A dictionary containing the current tokenized frame, next tokenized frame, and action. + """ + frame_indices = self.valid_actions_indices[i] + return ActionControlRow( + current_frame=self.tokenized_frames[frame_indices], # shape: (1, H, W) + next_frame=self.tokenized_frames[frame_indices + 1], # shape: (1, H, W) + action=self.actions[frame_indices], # shape: (7,) + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_finetuning.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_finetuning.py new file mode 100644 index 00000000..abaa7d35 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/action_control_finetuning.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from argparse import ArgumentParser + +import torch +from huggingface_hub import snapshot_download +from lightning.pytorch.loggers import WandbLogger +from megatron.core.dist_checkpointing.validation import StrictHandling +from megatron.core.optimizer import OptimizerConfig +from nemo import lightning as nl +from nemo.collections import llm +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + +from cosmos1.models.autoregressive.nemo.cosmos_action_control import ( + CosmosActionControlModel, + CosmosConfigActionControl5B, + CosmosConfigActionControl13B, +) +from cosmos1.models.autoregressive.nemo.post_training.action_control.action_control_ar_dataset import ( + ActionControlDataModule, +) + + +def main(args): + if "5B" in args.model_path or "5b" in args.model_path: + model_config = CosmosConfigActionControl5B() + elif "13B" in args.model_path or "13b" in args.model_path: + model_config = CosmosConfigActionControl13B() + else: + raise NotImplementedError + + if args.model_path in [ + "nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", + "nvidia/Cosmos-1.0-Autoregressive-13B-Video2World", + ]: + args.model_path = os.path.join(snapshot_download(args.model_path, allow_patterns=["nemo/*"]), "nemo") + + model = CosmosActionControlModel(model_config) + + data_module = ActionControlDataModule( + model_config=model_config, + global_batch_size=args.global_batch_size, + micro_batch_size=args.micro_batch_size, + tokenizer=None, + num_workers=15, + ) + + # Finetune is the same as train (Except train giverm the option to set tokenizer to None) + # So we use it since in this case we dont store a tokenizer with the model + llm.api.train( + model=model, + data=data_module, + trainer=nl.Trainer( + devices=args.tensor_model_parallel_size, + num_nodes=args.num_nodes, + max_steps=args.max_steps, + accelerator="gpu", + strategy=nl.MegatronStrategy( + tensor_model_parallel_size=args.tensor_model_parallel_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + pipeline_dtype=torch.bfloat16, + ckpt_load_strictness=StrictHandling.LOG_UNEXPECTED, + ), + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + num_sanity_val_steps=0, + limit_val_batches=0, + max_epochs=args.max_epochs, + log_every_n_steps=1, + callbacks=[ + ModelCheckpoint( + monitor="reduced_train_loss", + filename="{epoch}-{step}", + every_n_train_steps=args.save_every_n_steps, + save_top_k=2, + ), + PreemptionCallback(), + ], + ), + log=nl.NeMoLogger(wandb=(WandbLogger() if "WANDB_API_KEY" in os.environ else None), log_dir=args.log_dir), + optim=nl.MegatronOptimizerModule( + config=OptimizerConfig( + lr=args.lr, + bf16=True, + params_dtype=torch.bfloat16, + use_distributed_optimizer=False, + ) + ), + tokenizer=None, + resume=nl.AutoResume( + restore_config=RestoreConfig(path=args.model_path), + resume_if_exists=True, + resume_ignore_no_checkpoint=False, + resume_past_end=True, + ), + ) + + +if __name__ == "__main__": + parser = ArgumentParser() + # parser.add_argument("--data_path", required=True, type=str, help="The path to the .pt files") + parser.add_argument( + "--model_path", + default="nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", + type=str, + help="The path to the nemo model", + ) + + parser.add_argument("--log_dir", default="./log_dir", type=str, help="The path to the logs") + parser.add_argument("--tensor_model_parallel_size", default=4, type=int, help="Tensor model parallel size") + parser.add_argument("--num_nodes", default=1, type=int, help="Number of nodes") + parser.add_argument("--max_steps", default=10, type=int, help="The max number of steps to run finetuning") + parser.add_argument("--save_every_n_steps", default=5, type=int, help="How often to save a checkpoint") + parser.add_argument("--global_batch_size", default=1, type=int, help="The global batch size") + parser.add_argument( + "--micro_batch_size", default=1, type=int, help="The micro batch size if using pipeline parallel" + ) + parser.add_argument("--lr", default=5e-5, type=float, help="The learning rate") + parser.add_argument("--max_epochs", default=10, type=int, help="Max number of epochs") + + args = parser.parse_args() + + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/prepare_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/prepare_dataset.py new file mode 100644 index 00000000..7251a65b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/action_control/prepare_dataset.py @@ -0,0 +1,311 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import json +import logging +import os +from enum import Enum +from pathlib import Path +from typing import Literal, TypedDict + +import huggingface_hub +import numpy as np +import pooch +import torch +import torchvision +import typer +from einops import rearrange +from tqdm import tqdm + +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQJITTokenizer + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def create_tokenizer(tokenizer_tag: str = "nvidia/Cosmos-1.0-Tokenizer-DV8x16x16"): + """Creates a DiscreteVideoFSQJITTokenizer from a Hugging Face Hub tokenizer tag.""" + + tokenizer_path = Path(huggingface_hub.snapshot_download(tokenizer_tag)) + + # In the action-control finetuning task, we predict next frames as a function of the previous + # frame and a action vector. We set the pixel_chunk_duration to 1 to indicate that we're + # tokenizing indidivual still frames rather than entire videos. + video_tokenizer = DiscreteVideoFSQJITTokenizer( + enc_fp=str(tokenizer_path / "encoder.jit"), + dec_fp=str(tokenizer_path / "decoder.jit"), + name="discrete_video_fsq", + pixel_chunk_duration=1, + ).cuda() + + video_tokenizer.latent_chunk_duration = 1 + + return video_tokenizer + + +def download_bridge_data() -> Path: + """Downloads the bridge dataset (if not available in the hf cache) and extracts it to the local filesystem.""" + + logger.info("Downloading bridge dataset...") + + ds_path = huggingface_hub.cached_assets_path("cosmos", namespace="action-control", subfolder="datasets") + if (ds_path / "039134bac1ecbf2f26fe22b3a32078d9-bridge_train_data.tar.gz.untar").exists(): + logger.info("Bridge dataset already downloaded, skipping download.") + return ds_path / "039134bac1ecbf2f26fe22b3a32078d9-bridge_train_data.tar.gz.untar" + + processor = pooch.Untar() + _ = pooch.retrieve( + url="https://lf-robot-opensource.bytetos.com/obj/lab-robot-public/opensource_IRASim_v1/bridge_train_data.tar.gz", + known_hash=None, # Open-source dataset hash known to change sometimes. + processor=processor, + path=ds_path, + progressbar=True, + ) + # Validating the hash of the downloaded file can take a while, so we skip it here. If you want + # to validate the hash, uncomment the known_hash line above and remove the known_hash=None line. + logger.info("Bridge dataset downloaded successfully.") + + return Path(processor.extract_dir) # type: ignore + + +class ClipDataset(torch.utils.data.Dataset): + """A dataset of clips in the IRASim bridge structure, where each clip is identified by an integer ID.""" + + def __init__( + self, + bridge_data_root_dir: str | os.PathLike, + split: Literal["train", "val", "test"] = "train", + ): + """Initializes the dataset. + + Args: + bridge_data_root_dir: The root directory of the bridge dataset. + split: The split of the dataset to be used. + """ + + video_files = (Path(bridge_data_root_dir) / "videos" / split).glob("*/rgb.mp4") + self.split = split + self.bridge_data_root_dir = Path(bridge_data_root_dir) + self.clip_ids = sorted([int(path.parent.stem) for path in video_files]) + + def __len__(self) -> int: + return len(self.clip_ids) + + +class VideoDataset(ClipDataset): + """A dataset to decode video frames from the IRASim bridge dataset.""" + + def __getitem__(self, i: int) -> torch.Tensor: + """Gets all the frames from a single video clip by its ID. + + Args: + i: The index of the clip in the dataset. + + Returns: + A tensor of shape (T, C, 1, H, W) containing the video frames. + """ + + video_file = self.bridge_data_root_dir / "videos" / self.split / f"{self.clip_ids[i]}" / "rgb.mp4" + + # We use torchvision to perform the mp4 decoding, which means this operation is CPU-bound. + # By placing this in a dataset class and subsequently using a multiprocess dataloader, we can attempt to + # make this operation not the bottleneck in video tokenization. + video, _, _ = torchvision.io.read_video(str(video_file), pts_unit="sec") + + # Here we rearrange the video tensor from a single video clip to a batch of individual + # frames, by moving the temporal dimension to the batch dimension and subsequently using a + # collate function that concatenates the frames. + frames = rearrange(video, "t h w c -> t c 1 h w") + return frames + + +class AnnotationBatch(TypedDict): + """A single parsed clip's annotations""" + + state: torch.Tensor + action: torch.Tensor + continuous_gripper_state: torch.Tensor + + +class AnnotationDataset(ClipDataset): + """A dataset to load annotations, primarily the action vector, from the IRASim bridge dataset.""" + + def __getitem__(self, i: int) -> AnnotationBatch: + """Gets the annotations for a single video clip by its ID. + + Args: + i: The index of the clip in the dataset. + + Returns: + A dictionary containing the state, action, and continuous gripper state annotations. + """ + + state_file = self.bridge_data_root_dir / "videos" / self.split / f"{self.clip_ids[i]}" / "state.npy" + annotation_file = self.bridge_data_root_dir / "annotation" / self.split / f"{self.clip_ids[i]}.json" + with open(annotation_file, "rt") as f: + annotations = json.load(f) + + # Before returning the action tensor, we append a row of NaNs to the end of the tensor, + # since the action here represents the movement of the arm between frame i and frame i+1. + # The action tensor is therefore 1 row shorter than the number of frames in the video, and + # adding these NaNs lets us easily align these tensors and find valid training samples. + return { + "state": torch.Tensor(np.load(state_file)), + "action": torch.cat([torch.Tensor(annotations["action"]), np.nan * torch.ones((1, 7))], dim=0), + "continuous_gripper_state": torch.Tensor(annotations["continuous_gripper_state"]), + } + + +def dict_cat_collate(batch: list[AnnotationBatch]) -> AnnotationBatch: + """Collate function to concatenate the annotations in a batch of clips.""" + + return { + "state": torch.cat([d["state"] for d in batch], dim=0), + "action": torch.cat([d["action"] for d in batch], dim=0), + "continuous_gripper_state": torch.cat([d["continuous_gripper_state"] for d in batch], dim=0), + } + + +def get_annotations( + dataset_dir: Path, + dataset_split: Literal["train", "val", "test"], + batch_size: int, + num_workers: int, +) -> AnnotationBatch: + """Saves the annotations from the IRASim bridge dataset to the local filesystem.""" + + dataset = AnnotationDataset(dataset_dir, dataset_split) + dataloader = torch.utils.data.DataLoader( + dataset, + batch_size=batch_size, + num_workers=num_workers, + collate_fn=dict_cat_collate, + ) + + return dict_cat_collate(list(dataloader)) + + +def get_tokenized_frames( + dataset_dir: Path, + dataset_split: Literal["train", "val", "test"], + tokenizer_tag: str, + batch_size: int, + num_workers: int, +): + """Tokenizes the video frames from the IRASim bridge dataset and saves them to the local filesystem.""" + + video_tokenizer = create_tokenizer(tokenizer_tag) + dataset = VideoDataset(dataset_dir, dataset_split) + dataloader = torch.utils.data.DataLoader( + dataset, + num_workers=num_workers, + batch_size=batch_size, + collate_fn=lambda x: torch.concat(x, dim=0), + ) + + def iter_tokenized_batches(): + for batch in tqdm(dataloader): + # Convert to bf16 and normalize from [0, 255] to [-1, 1] + batch = batch.to(device="cuda", dtype=torch.bfloat16, non_blocking=True) / 127.5 - 1.0 + _, indices = video_tokenizer.encode(batch, pixel_chunk_duration=None) + yield indices.detach().to("cpu") + + # Here we're not being very memory efficient, as we're storing all the tokenized frames in + # memory before saving them to disk. For the bridge dataset with the 16x16 tokenizer, this + # results in only about a 4 Gb object. + with torch.no_grad(): + return torch.cat(list(iter_tokenized_batches()), dim=0) + + +class Split(str, Enum): + train = "train" + val = "val" + test = "test" + + +def get_default_output_prefix(dataset_split: Literal["train", "val", "test"], subfolder: str | None = None) -> Path: + """Returns the default directory for serializing the bridge dataset. + + Args: + dataset_split: The split of the dataset to be processed. + subfolder: The subfolder to use in HF_HOME/assets/cosmos/action-control. If not provided, the default + subfolder "autoregressive" will be used. + """ + if subfolder is None: + subfolder = "autoregressive" + + output_path = ( + huggingface_hub.cached_assets_path("cosmos", namespace="action-control", subfolder=subfolder) + / "bridge" + / dataset_split + ) + output_path.mkdir(parents=True, exist_ok=True) + return output_path + + +def main( + tokenizer_tag: str = "nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", + output_prefix: str | None = None, + dataset_split: Split = Split.train, + dataset_dir: str | None = None, + batch_size: int = 10, + num_workers: int = 10, +): + """Prepare the bridge dataset for autoregressive post-training. + + This script will download the bridge dataset from lf-robot-opensource.bytetos.com, extract the annotations and + video frames, and tokenize the video frames using the specified tokenizer. The resulting files will be saved + to the local filesystem. + + Args: + tokenizer_tag: The tag of the tokenizer to be used in tokenizing the video frames. + output_prefix: The prefix to be used for the output files. If omitted, the output files will be saved to + the huggingface cache directory. + dataset_split: The split of the dataset to be processed. + dataset_dir: The path to the extracted contents of bridge_train_data.tar.gz, in the format provided by IRASim. + If omitted, the ~30Gb dataset will be downloaded from lf-robot-opensource.bytetos.com. + batch_size: The batch size (number of clips) to use during tokenization. + num_workers: The number of worker processes to use when processing the dataset. + """ + + if dataset_dir is None: + dataset_path = download_bridge_data() / "opensource_robotdata" / "bridge" + else: + dataset_path = Path(dataset_dir) + + if output_prefix is None: + output_path = get_default_output_prefix(dataset_split.value) + else: + output_path = Path(output_prefix) + + logger.info("Serializing annotations...") + all_annotations = get_annotations(dataset_path, dataset_split.value, batch_size, num_workers) + + logger.info("Serializing tokenized frames...") + all_tokenized_frames = get_tokenized_frames( + dataset_path, dataset_split.value, tokenizer_tag, batch_size, num_workers + ) + + torch.save(all_annotations["state"], output_path / "state.pt") + torch.save(all_annotations["action"], output_path / "actions.pt") + torch.save(all_annotations["continuous_gripper_state"], output_path / "gripper.pt") + torch.save(all_tokenized_frames, output_path / "tokenized-frames.pt") + + +if __name__ == "__main__": + typer.run(main) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/general.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/general.py new file mode 100644 index 00000000..bb48fc5e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/general.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from argparse import ArgumentParser + +import torch +from huggingface_hub import snapshot_download +from lightning.pytorch.loggers import WandbLogger +from megatron.core.optimizer import OptimizerConfig +from nemo import lightning as nl +from nemo.collections import llm +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + +from cosmos1.models.autoregressive.nemo.cosmos import CosmosConfig4B, CosmosConfig12B, CosmosModel + + +def main(args): + if "4B" in args.model_path: + config = CosmosConfig4B() + elif "12B" in args.model_path: + config = CosmosConfig12B() + else: + raise NotImplementedError() + + if args.model_path in ["nvidia/Cosmos-1.0-Autoregressive-4B", "nvidia/Cosmos-1.0-Autoregressive-12B"]: + args.model_path = os.path.join(snapshot_download(args.model_path, allow_patterns=["nemo/*"]), "nemo") + + model = CosmosModel(config) + + valid_subfolders = [] + if os.path.exists(os.path.join(args.data_path, ".bin")) and os.path.exists(os.path.join(args.data_path, ".idx")): + valid_subfolders.append(args.data_path + "/") + else: + for subfolder in sorted(os.listdir(args.data_path)): + if not os.path.exists(os.path.join(args.data_path, subfolder, ".bin")) or not os.path.exists( + os.path.join(args.data_path, subfolder, ".idx") + ): + continue + valid_subfolders.append(os.path.join(args.data_path, subfolder) + "/") + + data_module = llm.PreTrainingDataModule( + paths=valid_subfolders, + seq_length=5 * 40 * 64, + global_batch_size=args.global_batch_size, + micro_batch_size=args.micro_batch_size, + tokenizer=None, + split=args.split_string, + num_workers=1, + index_mapping_dir=args.index_mapping_dir, + ) + + # Finetune is the same as train (Except train gives the option to set tokenizer to None) + # So we use it since in this case we dont store a tokenizer with the model + llm.api.train( + model=model, + data=data_module, + trainer=nl.Trainer( + devices=args.tensor_model_parallel_size, + num_nodes=1, + max_steps=args.max_steps, + accelerator="gpu", + strategy=nl.MegatronStrategy( + tensor_model_parallel_size=args.tensor_model_parallel_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + pipeline_dtype=torch.bfloat16, + ), + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + num_sanity_val_steps=0, + limit_val_batches=0, + max_epochs=args.max_epochs, + log_every_n_steps=1, + callbacks=[ + ModelCheckpoint( + monitor="reduced_train_loss", + filename="{epoch}-{step}", + every_n_train_steps=args.save_every_n_steps, + save_top_k=2, + ), + PreemptionCallback(), + ], + ), + log=nl.NeMoLogger(wandb=(WandbLogger() if "WANDB_API_KEY" in os.environ else None), log_dir=args.log_dir), + optim=nl.MegatronOptimizerModule( + config=OptimizerConfig( + lr=args.lr, + bf16=True, + params_dtype=torch.bfloat16, + use_distributed_optimizer=False, + ) + ), + tokenizer=None, + resume=nl.AutoResume( + restore_config=RestoreConfig(path=args.model_path), + resume_if_exists=True, + resume_ignore_no_checkpoint=False, + resume_past_end=True, + ), + ) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--data_path", required=True, type=str, help="The path to the .bin .idx files") + parser.add_argument( + "--model_path", default="nvidia/Cosmos-1.0-Autoregressive-4B", type=str, help="The path to the nemo model" + ) + parser.add_argument( + "--index_mapping_dir", default="./index_mapping", type=str, help="The directory to store mapped indices" + ) + parser.add_argument("--log_dir", default="./log_dir", type=str, help="The path to the logs") + parser.add_argument("--split_string", default="98,1,1", type=str, help="The train/test/validation split") + parser.add_argument("--tensor_model_parallel_size", default=2, type=int, help="Tensor model parallel size") + parser.add_argument("--max_steps", default=100, type=int, help="The max number of steps to run finetuning") + parser.add_argument("--save_every_n_steps", default=100, type=int, help="How often to save a checkpoint") + parser.add_argument("--global_batch_size", default=2, type=int, help="The global batch size") + parser.add_argument( + "--micro_batch_size", default=1, type=int, help="The micro batch size if using pipeline parallel" + ) + parser.add_argument("--lr", default=5e-5, type=float, help="The learning rate") + parser.add_argument("--max_epochs", default=10, type=int, help="Max number of epochs") + + args = parser.parse_args() + + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/prepare_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/prepare_dataset.py new file mode 100644 index 00000000..8d41c76e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/prepare_dataset.py @@ -0,0 +1,218 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +import re +from argparse import ArgumentParser +from glob import glob + +import torch +from einops import rearrange +from huggingface_hub import snapshot_download +from nemo.collections.nlp.data.language_modeling.megatron import indexed_dataset + +from cosmos1.models.autoregressive.nemo.utils import read_input_videos +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQJITTokenizer +from cosmos1.utils import log + + +CHUNK_SIZE = 250 # Number of videos per chunk + +NUM_GPUS = torch.cuda.device_count() + + +def main(args): + # -------------------------------------------------------------------------- + # 1) Initialize distributed (if launched with torchrun). + # If you plan to run single-GPU only, you could skip this entirely. + # -------------------------------------------------------------------------- + # Local rank is used for picking the GPU (e.g., rank 0 -> GPU 0, rank 1 -> GPU 1, etc.) + rank = int(os.environ["RANK"]) + local_rank = int(os.environ["LOCAL_RANK"]) % NUM_GPUS + world_size = int(os.environ["WORLD_SIZE"]) + device = torch.device("cuda", local_rank) + torch.cuda.set_device(device) + + # Download the tokenizer JIT models to local cache if needed + if args.encoder_path.startswith("nvidia/"): + args.encoder_path = os.path.join(snapshot_download(args.encoder_path), "encoder.jit") + if args.decoder_path.startswith("nvidia/"): + args.decoder_path = os.path.join(snapshot_download(args.decoder_path), "decoder.jit") + + NUM_CONTEXT_FRAMES = int(args.num_context_frames) + # Validate tokenizer compression factor format + if not re.match(r"^\d+,\d+,\d+$", args.tokenizer_compression_factor): + raise ValueError("Invalid tokenizer_compression_factor format. Expected format like '8,16,16' or '4,8,8'") + TOKENIZER_COMPRESSION_FACTOR = [int(x) for x in args.tokenizer_compression_factor.split(",")] + # -------------------------------------------------------------------------- + # Instantiate the tokenizer and move it to the correct GPU + # -------------------------------------------------------------------------- + video_tokenizer = DiscreteVideoFSQJITTokenizer( + enc_fp=args.encoder_path, + dec_fp=args.decoder_path, + name="discrete_video_fsq", + pixel_chunk_duration=NUM_CONTEXT_FRAMES, + compression_ratio=TOKENIZER_COMPRESSION_FACTOR, + ).to(device) + + # -------------------------------------------------------------------------- + # Gather all mp4 filepaths and sort them + # -------------------------------------------------------------------------- + filepaths_final = sorted(glob(f"{args.input_videos_dir}/*.mp4"))[:10000] + total_files = len(filepaths_final) + if total_files == 0: + if rank == 0: # Print once + log.warning(f"No .mp4 files found in {args.input_videos_dir}.") + return + + if rank == 0: + log.info(f"Found {total_files} .mp4 files in {args.input_videos_dir}.") + log.info(f"Chunk size = {CHUNK_SIZE}, total chunks = {((total_files - 1) // CHUNK_SIZE) + 1}.") + + # -------------------------------------------------------------------------- + # Loop over chunks, but only process the chunks that match our rank + # Example: If there are 10 chunks total and world_size=4, + # rank 0 processes chunks 0,4,8 + # rank 1 processes chunks 1,5,9 + # rank 2 processes chunks 2,6 + # rank 3 processes chunks 3,7 + # -------------------------------------------------------------------------- + num_chunks = (total_files - 1) // CHUNK_SIZE + 1 + log.info(f"World size: {world_size}, rank: {rank}, num_chunks: {num_chunks}") + for chunk_idx in range(num_chunks): + # Skip chunks not assigned to this rank + if chunk_idx % world_size != rank: + continue + # print amount of GPU memory used + start_index = chunk_idx * CHUNK_SIZE + chunk_filepaths = filepaths_final[start_index : start_index + CHUNK_SIZE] + + # Prepare output directory for this chunk + chunk_dir = os.path.join(args.output_prefix, str(chunk_idx)) if num_chunks > 1 else (args.output_prefix + "/") + bin_file = os.path.join(chunk_dir, ".bin") + idx_file = os.path.join(chunk_dir, ".idx") + + # Skip if dataset already exists + if os.path.exists(bin_file) and os.path.exists(idx_file): + log.info(f"[Rank {rank}] Skipping chunk {chunk_idx}, data already exists.") + continue + + if os.path.exists(bin_file): + log.info(f"[Rank {rank}] Deleting existing bin file {bin_file}") + os.remove(bin_file) + elif os.path.exists(idx_file): + log.info(f"[Rank {rank}] Deleting existing idx file {idx_file}") + os.remove(idx_file) + + # ---------------------------------------------------------------------- + # Create a new builder for this chunk + # ---------------------------------------------------------------------- + os.makedirs(chunk_dir, exist_ok=True) + builder = indexed_dataset.make_builder( + bin_file, + impl="mmap", + chunk_size=64, + pad_id=0, + retrieval_db=None, + vocab_size=64000, + stride=64, + ) + + log.info(f"[Rank {rank}] Processing chunk {chunk_idx}, number of files = {len(chunk_filepaths)}") + + for idx_in_chunk, filepath in enumerate(chunk_filepaths, 1): + try: + # Move the video input to GPU + input_video = read_input_videos( + filepath, data_resolution=[int(args.height), int(args.width)], num_frames=NUM_CONTEXT_FRAMES + ).to(device) + input_video = input_video[:, :, :NUM_CONTEXT_FRAMES, :, :] + + batch_size, channels, frames, height, width = input_video.shape + latent_shape = ( + (frames - 1) // TOKENIZER_COMPRESSION_FACTOR[0] + 1, + height // TOKENIZER_COMPRESSION_FACTOR[1], + width // TOKENIZER_COMPRESSION_FACTOR[2], + ) + T, H, W = latent_shape + + # Temporarily update the tokenizer's latent_chunk_duration + video_tokenizer.latent_chunk_duration = T + + quantized_out, _ = video_tokenizer.encode(input_video, pixel_chunk_duration=None) + indices = video_tokenizer.fsq_quantizer.codes_to_indices(quantized_out.permute(0, 2, 3, 4, 1)) + + # Flatten to 1D + indices = rearrange(indices, "B T H W -> (B T H W)").detach().cpu() + builder.add_item(torch.IntTensor(indices)) + builder.end_document() + + log.info( + f"[Rank {rank}] Chunk {chunk_idx}, file {idx_in_chunk}/{len(chunk_filepaths)}: " + f"{os.path.basename(filepath)} processed successfully." + ) + + except Exception as e: + log.error(f"[Rank {rank}] Error processing {filepath}: {e}") + + # Finalize .idx and .bin for this chunk + builder.finalize(idx_file) + log.info(f"[Rank {rank}] Stored .bin and .idx files in {chunk_dir}") + + if rank == 0: + log.info("All ranks have finished processing.") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--input_videos_dir", required=True, type=str, help="Path to the input .mp4 files") + parser.add_argument("--width", default=1024, type=int, help="Width of the input videos") + parser.add_argument("--height", default=640, type=int, help="Height of the input videos") + parser.add_argument( + "--encoder_path", + default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", + type=str, + help="Hugging Face repo or local path to encoder", + ) + parser.add_argument( + "--decoder_path", + default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", + type=str, + help="Hugging Face repo or local path to decoder", + ) + parser.add_argument( + "--num_context_frames", + default=33, + type=int, + help="Number of context frames to use for the tokenizer", + ) + parser.add_argument( + "--tokenizer_compression_factor", + default="8,16,16", + type=str, + help="Tokenizer compression factor", + ) + parser.add_argument( + "--output_prefix", + required=True, + type=str, + help="Directory to write chunked .idx and .bin files (e.g. /path/to/output)", + ) + args = parser.parse_args() + + with torch.no_grad(): + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/tokenizer/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/tokenizer/README.md new file mode 100644 index 00000000..aecb6e8f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/tokenizer/README.md @@ -0,0 +1,300 @@ +# Cosmos Autoregressive-based World Foundation Models: Changing the Video Tokenizer + +Learn how to post-train Cosmos Autoregressive-based World Foundation Models (WFMs) using the NVIDIA NeMo Framework when swapping out the original tokenizer for a new one. This recipe provides a conceptual overview of how to adapt your existing model to handle a different discrete video (DV) tokenizer, ensuring high-quality video generation under a new compression setting. + +## Model Support Matrix + +The NeMo Framework supports the following Cosmos Autoregressive (AR) models for post-training. Choose the model that best suits your compute and memory budget. + +| Model Name | Model Status | Compute Requirements for Post-Training | +|-------------------------------------------|--------------|------------------------------------------| +| Cosmos-1.0-Autoregressive-4B | Supported | 2 NVIDIA GPUs* | +| Cosmos-1.0-Autoregressive-12B | Supported | 8 NVIDIA GPUs* | +| Cosmos-1.0-Autoregressive-5B-Video2World | Supported | 2 NVIDIA GPUs* | +| Cosmos-1.0-Autoregressive-13B-Video2World | Supported | 8 NVIDIA GPUs* | + +*Either H100-80GB or A100-80GB GPUs are recommended. + +## Prerequisites + +1. **Review General Requirements** + - **GPU/Driver:** Ensure you have sufficient GPU memory (see table above). + - **Containerization:** Use Docker with NVIDIA Container Runtime or a compatible environment. + - **Hugging Face Access Token:** Required for downloading Cosmos checkpoints. + - **(Optional) Weights & Biases:** For experiment tracking. + +2. **Clone the Cosmos Repository** + + ```bash + git clone https://github.com/NVIDIA/Cosmos.git + ``` + +3. **Start the NeMo Framework Container** + + The official [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos AR models. Ensure you have Docker installed along with the NVIDIA Container Runtime. Replace `$PATH_TO_COSMOS_REPO` with the path to your local clone. + + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidia/nemo:25.02rc3 bash + ``` + +4. **Install Python Dependencies** + + Inside the container (or your environment), install the following Python packages: + + ```bash + pip install -r requirements.txt + ``` + +## Why Change Tokenizers? + +Cosmos Autoregressive models are typically trained on a specific tokenizer configuration (e.g., 8×16×16). If you’d like to reduce patch size or change compression (e.g., to 4×8×8), you can post-train the existing weights so that the model effectively aligns its internal representations with the new token embeddings—without re-training the tokenizer. + +## Tutorial: Finetuning Cosmos-4B on 10k Videos with a New Tokenizer + +In this tutorial, we will: +- Take a model originally trained on an 8×16×16 tokenizer. +- Post-train it on a 4×8×8 tokenizer. +- Demonstrate using a sample dataset of 10 (in production, we recommend using 10k videos from a distribution similar to that of pretraining). + +### 1. Calculate Sequence Lengths + +- **Original Sequence Length:** 12,800 tokens (33 frames * 640 px width * 1024 px height, when tokenized with an 8×16×16 tokenizer becomes ⌈33/8⌉ * (640/16) * (1024/16) = 5 * 64 * 80 = 12,800 tokens) +- **New Sequence Length:** 12,800 tokens (17 frames * 320 px width * 512 px height, when tokenized with a 4×8×8 tokenizer becomes ⌈17/8⌉ * (320/4) * (512/8) = 5 * 80 * 64 = 12,800 tokens) +- For other resolutions or frame counts, recalculate your maximum tokens to ensure you do not exceed the model’s capacity. + +```bash +export WIDTH=512 +export HEIGHT=320 +export NUM_FRAMES=17 +export TOKENIZER_COMPRESSION_FACTOR=4,8,8 +export NUM_GPUS=8 # change this to your number of GPUs + +export ENCODER_PATH="nvidia/Cosmos-0.1-Tokenizer-DV4x8x8" +export DECODER_PATH="nvidia/Cosmos-0.1-Tokenizer-DV4x8x8" +export INPUT_VIDEOS_DIR="cosmos1/models/autoregressive/assets/v1p0/batch_inputs" +export OUTPUT_TOKENS_DIR="./my_4x8x8_tokens" + +mkdir -p ./experiments +``` + +### 2. Prepare the Dataset + +Ensure that you have all the videos available in the sample dataset. + +``` +git lfs fetch --all +``` + +Run the following command to prepare the dataset. + +```bash +torchrun --nproc-per-node=1 cosmos1/models/autoregressive/nemo/post_training/prepare_dataset.py \ + --width $WIDTH \ + --height $HEIGHT \ + --num_context_frames $NUM_FRAMES \ + --tokenizer_compression_factor $TOKENIZER_COMPRESSION_FACTOR \ + --encoder_path $ENCODER_PATH \ + --decoder_path $DECODER_PATH \ + --input_videos_dir $INPUT_VIDEOS_DIR \ + --output_prefix $OUTPUT_TOKENS_DIR +``` + +### 3. Training Recipe & Hyperparameters + +This tutorial offers a baseline recipe to help you build intuition about post-training. You may tweak these parameters based on your domain, hardware, or convergence requirements. For production, consider: + +- **Number of Videos:** 10,000 +- **Global Batch Size:** 32 +- **Learning Rate:** 1e-4 +- **Recommended Steps:** ~300 steps (roughly 1 epoch for ~128M tokens) +- **Loss Convergence:** Target loss in the range of 6-7 + +A good rule of thumb: +- Monitor training/validation loss to ensure steady improvement. +- Evaluate sample outputs at various checkpoints to verify coherent generation. +- For more precision, consider running additional training steps. + +In this tutorial, we will only post-train on 10 sample videos. + +### 4. Run Post-training + +Use the NeMo post-training or fine-tuning scripts (depending on your model variant) and point to your tokenized dataset. For example: + +```bash +torchrun --nproc-per-node=$NUM_GPUS cosmos1/models/autoregressive/nemo/post_training/general.py \ + --data_path $OUTPUT_TOKENS_DIR \ + --split_string 10,1,1 \ + --log_dir ./experiments/example_log_dir \ + --global_batch_size 1 \ + --micro_batch_size 1 \ + --lr 1e-4 \ + --max_steps 100 --save_every_n_steps 25 \ + --max_epochs 1 \ + --tensor_model_parallel_size $NUM_GPUS \ + --model_path nvidia/Cosmos-1.0-Autoregressive-4B +``` + +- Adjust parameters such as `--max_steps`, `--global_batch_size`, and `--lr` to suit your needs. +- Ensure that `--model_path` matches the checkpoint originally trained on an 8×16×16 tokenizer. +- For an explanation on other configuration options, please see [the general post-training tutorial](https://github.com/NVIDIA/Cosmos/blob/main/cosmos1/models/autoregressive/nemo/post_training/README.md#configuration-options). + +### 5. Monitor Quality + +After training, monitor checkpoints and generate sample outputs. Please keep in mind that you will have to +disable the diffusion decoder, as this is not yet supported for non-standard tokenizers. + +1. **Checkpoints:** + Checkpoints are saved under `./experiments/example_log_dir/default/checkpoints/`. For example: + ```bash + epoch=1-step=49 + epoch=1-step=99 + epoch=1-step=149 + ``` + Choose a checkpoint (e.g., `epoch=1-step=99`) for inference. + +2. **Run Inference:** + Generate sample video outputs: + ```bash + # Set the checkpoint directory + export CKPT_DIR="./experiments/example_log_dir/default/checkpoints/epoch=1-step=99" + # Create an evaluation folder + mkdir -p "$CKPT_DIR/evals" + export INPUT_FILE="cosmos1/models/autoregressive/assets/v1p0/input.mp4" + git lfs fetch --all + torchrun --nproc-per-node=$NUM_GPUS \ + cosmos1/models/autoregressive/nemo/inference/general.py \ + --input_image_or_video_path $INPUT_FILE \ + --video_save_name "$CKPT_DIR/evals/generated.mp4" \ + --ar_model_dir $CKPT_DIR \ + --encoder_path $ENCODER_PATH \ + --decoder_path $DECODER_PATH \ + --disable_diffusion_decoder \ + --width $WIDTH \ + --height $HEIGHT \ + --num_context_frames $NUM_FRAMES \ + --tokenizer_compression_factor $TOKENIZER_COMPRESSION_FACTOR + ``` + +3. **(Optional) Evaluate with TokenBench:** + To compute quantitative metrics (e.g., PSNR, SSIM), generate a large number of videos using the script above and place them into the `evals` + folder of the checkpoint. Please ensure that the input filename in the original folder (e.g. `my-video.mp4` matches the output filename in the + `evals` folder). This is because PSNR and SSIM compare video quality against a reference, and our script will match the output video with + the same filename under the input folder. + + ```bash + # Install extra dependencies for TokenBench + pip install scikit-image imageio mediapy + export EVAL_REFERENCE_VIDEOS_DIR="..." # <-- Replace with your eval videos input directory + export EVAL_GENERATED_VIDEOS_DIR="$CKPT_DIR/evals" + + python cosmos1/models/autoregressive/nemo/post_training/tokenizer/token_bench.py \ + --gtpath $EVAL_REFERENCE_VIDEOS_DIR \ + --targetpath $EVAL_GENERATED_VIDEOS_DIR \ + --width $WIDTH \ + --num_frames $NUM_FRAMES \ + --recursive + ``` + **Note:** Ensure that the filenames in the reference and generated folders match and that you use a sample size of around 1,000 videos for robust evaluation. + +## Tutorial: Finetuning Cosmos-5B-Video2World with a New Tokenizer + +To fine-tune a Video2World model, begin by setting up the container and environment as described above in the tutorial for the Cosmos-4B base model. Please note, for Video2World fine-tuning, the data format +is slightly different. The data format used in the fine-tuning script is a `.jsonl` file, where each line has the format + +``` +{"prompt": "[example prompt: a video of a robotic arm]", "visual_input": "/path/to/video.mp4"} +``` + +Please prepare such a file using the dataset you would like to fine-tune on. Change the environment variable `INPUT_JSONL` to the path to this file. + +### 1. Environment Variables + +Set common environment variables for consistency: + +```bash +export WIDTH=512 +export HEIGHT=320 +export NUM_FRAMES=17 +export TOKENIZER_COMPRESSION_FACTOR=4,8,8 + +export ENCODER_PATH=nvidia/Cosmos-0.1-Tokenizer-DV4x8x8 +export DECODER_PATH=nvidia/Cosmos-0.1-Tokenizer-DV4x8x8 + +export NUM_GPUS=8 # Replace with the number of GPUs you have +export OUTPUT_TOKENS_DIR=./my_v2w_tokens # Replace with the directory to store your tokens +export INPUT_JSONL="cosmos1/models/autoregressive/assets/v1p0/batch_inputs/video2world.jsonl" # Replace with the filepath to your jsonl describing your dataset +``` + +### 2. Prepare the Video2World Dataset + +Use the specialized script to resize videos, tokenize frames, and organize your dataset: + +```bash +python cosmos1/models/autoregressive/nemo/post_training/video2world_prepare_dataset.py \ + --width $WIDTH \ + --height $HEIGHT \ + --num_context_frames $NUM_FRAMES \ + --tokenizer_compression_factor $TOKENIZER_COMPRESSION_FACTOR \ + --encoder_path $ENCODER_PATH \ + --decoder_path $DECODER_PATH \ + --output_dir $OUTPUT_TOKENS_DIR \ + --input_jsonl $INPUT_JSONL +``` + +### 3. Training/Finetuning for Video2World + +Fine-tune a Video2World–specific checkpoint (or any compatible Cosmos AR model) on your new data. For example, using the 5B Video2World model: + +```bash +torchrun --nproc-per-node=$NUM_GPUS cosmos1/models/autoregressive/nemo/post_training/video2world_finetuning.py \ + --model_path nvidia/Cosmos-1.0-Autoregressive-5B-Video2World \ + --data_path $OUTPUT_TOKENS_DIR \ + --save_every_n_steps 50 \ + --tensor_model_parallel_size $NUM_GPUS \ + --global_batch_size 1 \ + --micro_batch_size 1 \ + --lr 1e-4 \ + --max_steps 100 \ + --max_epochs 10 \ + --log_dir ./experiments/example_log_dir_v2w +``` + +*Tip:* Adjust `global_batch_size` and `micro_batch_size` based on your GPU capacity. + +### 4. Video2World Inference + +After training, run inference with the provided script to generate or reconstruct videos. Please keep in mind that you will have to +disable the diffusion decoder, as this is not yet supported for non-standard tokenizers. + +```bash +export CKPT_DIR="./experiments/example_log_dir_v2w/default/checkpoints/epoch=0-step=49" +export INPUT_VIDEO="cosmos1/models/autoregressive/assets/v1p0/batch_inputs/0.mp4" +export INPUT_PROMPT="A video recorded from a moving vehicle's perspective, capturing roads, buildings, landscapes, and changing weather and lighting conditions." + +git lfs pull $INPUT_VIDEO +mkdir -p "$CKPT_DIR/evals" + +python cosmos1/models/autoregressive/nemo/inference/video2world.py \ + --input_image_or_video_path "$INPUT_VIDEO" \ + --prompt "$INPUT_PROMPT" \ + --width $WIDTH \ + --height $HEIGHT \ + --num_context_frames $NUM_FRAMES \ + --video_save_name "$CKPT_DIR/evals/0.mp4" \ + --tokenizer_compression_factor $TOKENIZER_COMPRESSION_FACTOR \ + --encoder_path $ENCODER_PATH \ + --decoder_path $DECODER_PATH \ + --ar_model_dir $CKPT_DIR \ + --disable_diffusion_decoder +``` + +*Key Parameters:* +- `--input_image_or_video_path`: Source video for context. +- `--prompt`: Textual guidance for video generation or reconstruction. +- `--encoder_path`/`--decoder_path`: Must match the tokenizer settings used during training. + +### 5. (Optional) Evaluate with TokenBench + +Follow the steps from the Cosmos-4B tutorial to evaluate with TokenBench. The command will be the same. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/tokenizer/token_bench.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/tokenizer/token_bench.py new file mode 100644 index 00000000..a9fe0fda --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/tokenizer/token_bench.py @@ -0,0 +1,240 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +Combined script to process videos in two directories (reference and generated) +and then compute video quality metrics: PSNR and SSIM. + +Each video is clipped to 10 seconds (or at least 300 frames) and resized using the specified +number of frames (--num_frames) and width (--width), with the processed output written to a +"processed" subfolder in the original directory. + +Example usage: + pip install scikit-image imageio mediapy + python cosmos1/models/autoregressive/nemo/post_training/tokenizer/token_bench.py --gtpath /path/to/ref_videos --targetpath /path/to/gen_videos --width 320 --num_frames 17 --recursive +""" + +import argparse +import json +import os +from glob import glob + +import imageio + +# Keep mediapy for functions like resize_video and write_video +import mediapy as media +import numpy as np +from skimage.metrics import structural_similarity as ssim +from tqdm import tqdm + + +# -------------------------------------------------- +# New Video Reader Using imageio (Replaces media.read_video) +# -------------------------------------------------- +def read_video(filepath: str) -> np.ndarray: + """Reads a video file using imageio and returns a NumPy array of frames.""" + try: + reader = imageio.get_reader(filepath, "ffmpeg") + except Exception as e: + raise RuntimeError(f"Error reading video file {filepath}: {e}") + frames = [] + for frame in reader: + frames.append(frame) + return np.array(frames) + + +# ---------------------------- +# Video Processing Functions +# ---------------------------- +def resize_video(video: np.ndarray, num_frames: int = 17, short_size: int = None) -> np.ndarray: + """Resizes a video so that its shorter side is equal to `short_size` and clips to `num_frames` frames.""" + if short_size is None: + return video[:num_frames] + height, width = video.shape[-3:-1] + if height <= width: + height_new, width_new = short_size, int(width * short_size / height + 0.5) + else: + height_new, width_new = int(height * short_size / width + 0.5), short_size + return media.resize_video(video[:num_frames], shape=(height_new, width_new)) + + +def process_video_file(video_file: str, output_dir: str, num_frames: int, width: int) -> None: + """Process a single video file: clip to 10 seconds (or at least 300 frames) and resize using provided parameters.""" + try: + video_reader = imageio.get_reader(video_file) + except Exception as e: + print(f"Error reading {video_file}: {e}") + return + + video_frames = [] + for frame in video_reader: + video_frames.append(frame) + input_video = np.array(video_frames) + meta_data = video_reader.get_meta_data() + video_fps = meta_data["fps"] + T, H, W, C = input_video.shape + + # Clip the video to 10 seconds (or at least 300 frames) + num_frame_thres = max(int(np.ceil(video_fps * 10)), 300) + output_video = input_video[:num_frame_thres] if T > num_frame_thres else input_video + + output_video = resize_video(output_video, num_frames, width) + + # Write output as .mp4 regardless of input extension + base_name = os.path.splitext(os.path.basename(video_file))[0] + output_file = os.path.join(output_dir, base_name + ".mp4") + os.makedirs(os.path.dirname(output_file), exist_ok=True) + print("Writing processed video to", output_file) + media.write_video(output_file, output_video, fps=video_fps) + + +def process_directory( + input_dir: str, output_dir: str, recursive: bool = False, num_frames: int = 17, width: int = 320 +) -> None: + """Process all video files in input_dir (excluding those in a 'processed' folder) + and write the processed videos to output_dir.""" + video_extensions = ["mp4", "mov", "avi", "mkv"] + input_files = [] + if recursive: + for ext in video_extensions: + pattern = os.path.join(input_dir, "**", f"*.{ext}") + input_files.extend(glob(pattern, recursive=True)) + else: + for ext in video_extensions: + pattern = os.path.join(input_dir, f"*.{ext}") + input_files.extend(glob(pattern)) + + # Exclude files that are already in a 'processed' subdirectory. + input_files = [f for f in input_files if "processed" not in os.path.normpath(f).split(os.sep)] + + input_files = sorted(input_files) + print(f"Found {len(input_files)} videos in {input_dir} to process.") + for video_file in input_files: + process_video_file(video_file, output_dir, num_frames, width) + + +# ---------------------------- +# Metric Computation Functions +# ---------------------------- +_FLOAT32_EPS = np.finfo(np.float32).eps +_UINT8_MAX_F = float(np.iinfo(np.uint8).max) + + +def PSNR(input0: np.ndarray, input1: np.ndarray) -> float: + """Compute PSNR between two videos or images.""" + assert input0.shape == input1.shape, "inputs should have the same shape" + mse = ((input0 - input1) ** 2).mean() + psnr = 20 * np.log10(_UINT8_MAX_F / (np.sqrt(mse) + _FLOAT32_EPS)) + return psnr.item() + + +def SSIM(input0: np.ndarray, input1: np.ndarray) -> float: + """Compute SSIM between two videos or images.""" + assert input0.shape == input1.shape, "inputs should have the same shape" + # If a single video/image, wrap it in an array for uniformity. + if input0.ndim == 3: + input0, input1 = np.array([input0]), np.array([input1]) + from concurrent.futures import ThreadPoolExecutor + + def compute_ssim(pair): + one_image0, one_image1 = pair + return ssim(one_image0, one_image1, data_range=_UINT8_MAX_F, multichannel=True, channel_axis=-1) + + with ThreadPoolExecutor() as executor: + ssim_values = list(executor.map(compute_ssim, zip(input0, input1))) + return np.mean(ssim_values) + + +def main_psnr_ssim() -> None: + # Build dictionaries mapping file basename to full path. + gt_files = {os.path.basename(f): f for f in glob(os.path.join(args.gtpath, f"*.{args.ext}"))} + target_files = {os.path.basename(f): f for f in glob(os.path.join(args.targetpath, f"*.{args.ext}"))} + + # Files that are common in both directories. + included_keys = sorted(set(gt_files.keys()) & set(target_files.keys())) + # Files that exist only in one directory. + skipped_keys = sorted(set(gt_files.keys()).symmetric_difference(set(target_files.keys()))) + + print(f"Included: {', '.join(included_keys)} | Skipped: {', '.join(skipped_keys)}") + + if not included_keys: + print("No matching files found.") + return + + psnr_values, ssim_values = [], [] + print(f"Calculating PSNR and SSIM on {len(included_keys)} pairs ...") + for key in tqdm(included_keys, total=len(included_keys)): + file0 = gt_files[key] + file1 = target_files[key] + vid0 = read_video(file0).astype(np.float32) + vid1 = read_video(file1).astype(np.float32) + psnr_val = PSNR(vid0, vid1) + ssim_val = SSIM(vid0, vid1) + psnr_values.append([key, float(psnr_val)]) + ssim_values.append([key, float(ssim_val)]) + print(f"{key} PSNR: {psnr_val:.3f}, SSIM: {ssim_val:.3f}") + + mean_psnr = np.mean([val for _, val in psnr_values]) + mean_ssim = np.mean([val for _, val in ssim_values]) + print(f"Mean PSNR: {mean_psnr:.3f}") + print(f"Mean SSIM: {mean_ssim:.3f}") + + with open(os.path.join(args.targetpath, "psnr.json"), "w") as fw: + json.dump(psnr_values, fw) + with open(os.path.join(args.targetpath, "ssim.json"), "w") as fw: + json.dump(ssim_values, fw) + + +# ---------------------------- +# Main Function and Argument Parsing +# ---------------------------- +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process two directories of videos (reference and generated) and compute PSNR and SSIM metrics." + ) + parser.add_argument( + "--gtpath", type=str, required=True, help="Path to the directory of reference (ground-truth) videos" + ) + parser.add_argument("--targetpath", type=str, required=True, help="Path to the directory of generated videos") + parser.add_argument("--ext", type=str, default="mp4", help="Video file extension (e.g., mp4)") + parser.add_argument("--device", type=str, default="cuda", help="Device to use for metrics computation") + parser.add_argument( + "--recursive", action="store_true", help="If set, process videos in subdirectories recursively" + ) + parser.add_argument("--width", type=int, default=320, help="Target width (shorter side) for video resizing") + parser.add_argument("--num_frames", type=int, default=17, help="Number of frames to keep for each processed video") + args = parser.parse_args() + + # Always process videos. + gt_processed = os.path.join(args.gtpath, "processed") + os.makedirs(gt_processed, exist_ok=True) + gen_processed = os.path.join(args.targetpath, "processed") + os.makedirs(gen_processed, exist_ok=True) + print("Processing reference videos ...") + process_directory( + args.gtpath, gt_processed, recursive=args.recursive, num_frames=args.num_frames, width=args.width + ) + print("Processing generated videos ...") + process_directory( + args.targetpath, gen_processed, recursive=args.recursive, num_frames=args.num_frames, width=args.width + ) + # Use the processed directories for metric computation. + args.gtpath = gt_processed + args.targetpath = gen_processed + + # Always compute PSNR and SSIM. + main_psnr_ssim() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_dataset.py new file mode 100644 index 00000000..a365118e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_dataset.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import json + +import torch +from nemo.collections.llm.gpt.data.mock import MockDataModule +from torch.utils.data import Dataset + +from cosmos1.models.autoregressive.modules.embedding import SinCosPosEmbAxisTE +from cosmos1.models.autoregressive.nemo.cosmos import CosmosConfig + + +TOKENIZER_COMPRESSION_FACTOR = [8, 16, 16] +DATA_RESOLUTION_SUPPORTED = [640, 1024] +NUM_CONTEXT_FRAMES = 33 +BOV_TOKEN = 64000 +PAD_ID = 64002 + + +class CosmosVideo2WorldDataset(Dataset): + def __init__(self, data_path, model_config, split="train"): + self.data_path = data_path + self.model_config = model_config + self.split = split + self.abs_pos_emb = get_abs_pos_embed(model_config, training_type="text_to_video") + metadata_file = f"{self.data_path}/metadata.json" + with open(metadata_file, "r") as f: + metadata = json.load(f) + self.metadata = metadata + + def __len__(self): + return self.metadata[f"{self.split}_samples"] + + def __getitem__(self, idx): + prompt_embedding = torch.load(f"{self.data_path}/{self.split}_prompt_{idx}.pt", map_location="cpu").to( + torch.bfloat16 + ) + video_tokens = torch.load(f"{self.data_path}/{self.split}_video_{idx}.pt", map_location="cpu").to( + torch.bfloat16 + ) + seq_len = video_tokens.numel() - 1 + assert seq_len == 12864, f"expected seq len 12864 but got {seq_len}" + attention_mask = ( + torch.tril(torch.ones((seq_len, seq_len), device=video_tokens.device)).unsqueeze(0).to(torch.bool) + ) + loss_mask = torch.cat([torch.ones(12800), torch.zeros(64)]) + sample = { + "tokens": video_tokens[:-1].to(torch.int64), + "position_ids": torch.arange(0, seq_len), + "attention_mask": attention_mask, + "labels": video_tokens[1:].to(torch.int64), + "abs_pos_embed": self.abs_pos_emb, + "loss_mask": loss_mask, + "context": prompt_embedding, + } + + return sample + + def _collate_fn(self, batch): + op = torch.utils.data.dataloader.default_collate(batch) + op["attention_mask"] = op["attention_mask"][0, :, :, :].unsqueeze(dim=0) + op["abs_pos_embed"] = op["abs_pos_embed"][0, :, :, :] + op["context"] = op["context"].permute(1, 0, 2) + return op + + def collate_fn(self, batch): + return self._collate_fn(batch) + + +def get_abs_pos_embed(model_config: CosmosConfig, training_type: str | None = "text_to_video"): + pos_emb = SinCosPosEmbAxisTE( + model_config.hidden_size, + latent_shape=model_config.latent_shape, + pad_to_multiple_of=model_config.pad_to_multiple_of, + device="cpu", + ) + abs_pos_emb = pos_emb.forward(training_type=training_type) + abs_pos_emb = abs_pos_emb.transpose(0, 1).contiguous() + return abs_pos_emb + + +class CosmosVideo2WorldDataModule(MockDataModule): + def __init__(self, *args, **kwargs): + data_path = kwargs["data_path"] + model_config = kwargs["model_config"] + del kwargs["data_path"] + del kwargs["model_config"] + super().__init__(*args, **kwargs) + self.dataset = CosmosVideo2WorldDataset + self.data_path = data_path + self.model_config = model_config + + def setup(self, stage: str = "") -> None: + self._train_ds = self.dataset(data_path=self.data_path, model_config=self.model_config, split="train") + self._validation_ds = self.dataset(data_path=self.data_path, model_config=self.model_config, split="test") + self._test_ds = self.dataset(data_path=self.data_path, model_config=self.model_config, split="val") diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_finetuning.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_finetuning.py new file mode 100644 index 00000000..d0b1c9d5 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_finetuning.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from argparse import ArgumentParser + +import torch +from huggingface_hub import snapshot_download +from lightning.pytorch.loggers import WandbLogger +from megatron.core.optimizer import OptimizerConfig +from nemo import lightning as nl +from nemo.collections import llm +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + +from cosmos1.models.autoregressive.nemo.cosmos_video2world import ( + CosmosConfigVideo2World5B, + CosmosConfigVideo2World13B, + CosmosVideo2WorldModel, +) +from cosmos1.models.autoregressive.nemo.post_training.video2world_dataset import CosmosVideo2WorldDataModule + + +def main(args): + if "5B" in args.model_path or "5b" in args.model_path: + model_config = CosmosConfigVideo2World5B() + elif "13B" in args.model_path or "13b" in args.model_path: + model_config = CosmosConfigVideo2World13B() + else: + raise NotImplementedError + + if args.model_path in [ + "nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", + "nvidia/Cosmos-1.0-Autoregressive-13B-Video2World", + ]: + args.model_path = os.path.join(snapshot_download(args.model_path, allow_patterns=["nemo/*"]), "nemo") + + model = CosmosVideo2WorldModel(model_config) + + data_module = CosmosVideo2WorldDataModule( + data_path=args.data_path, + model_config=model_config, + seq_length=12864, + global_batch_size=args.global_batch_size, + micro_batch_size=args.micro_batch_size, + tokenizer=None, + num_workers=1, + ) + + # Finetune is the same as train (Except train giverm the option to set tokenizer to None) + # So we use it since in this case we dont store a tokenizer with the model + llm.api.train( + model=model, + data=data_module, + trainer=nl.Trainer( + devices=args.tensor_model_parallel_size, + num_nodes=1, + max_steps=args.max_steps, + accelerator="gpu", + strategy=nl.MegatronStrategy( + tensor_model_parallel_size=args.tensor_model_parallel_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + sequence_parallel=False, + pipeline_dtype=torch.bfloat16, + ), + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + num_sanity_val_steps=0, + limit_val_batches=0, + max_epochs=args.max_epochs, + log_every_n_steps=1, + callbacks=[ + ModelCheckpoint( + monitor="reduced_train_loss", + filename="{epoch}-{step}", + every_n_train_steps=args.save_every_n_steps, + save_top_k=2, + ), + PreemptionCallback(), + ], + ), + log=nl.NeMoLogger(wandb=(WandbLogger() if "WANDB_API_KEY" in os.environ else None), log_dir=args.log_dir), + optim=nl.MegatronOptimizerModule( + config=OptimizerConfig( + lr=args.lr, + bf16=True, + params_dtype=torch.bfloat16, + use_distributed_optimizer=False, + ) + ), + tokenizer=None, + resume=nl.AutoResume( + restore_config=RestoreConfig(path=args.model_path), + resume_if_exists=True, + resume_ignore_no_checkpoint=False, + resume_past_end=True, + ), + ) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--data_path", required=True, type=str, help="The path to the .pt files") + parser.add_argument( + "--model_path", + default="nvidia/Cosmos-1.0-Autoregressive-5B-Video2World", + type=str, + help="The path to the nemo model", + ) + + parser.add_argument("--log_dir", default="./log_dir", type=str, help="The path to the logs") + parser.add_argument("--tensor_model_parallel_size", default=4, type=int, help="Tensor model parallel size") + parser.add_argument("--max_steps", default=10, type=int, help="The max number of steps to run finetuning") + parser.add_argument("--save_every_n_steps", default=5, type=int, help="How often to save a checkpoint") + parser.add_argument("--global_batch_size", default=1, type=int, help="The global batch size") + parser.add_argument( + "--micro_batch_size", default=1, type=int, help="The micro batch size if using pipeline parallel" + ) + parser.add_argument("--lr", default=5e-5, type=float, help="The learning rate") + parser.add_argument("--max_epochs", default=10, type=int, help="Max number of epochs") + + args = parser.parse_args() + + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_prepare_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_prepare_dataset.py new file mode 100644 index 00000000..eb4899a2 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/post_training/video2world_prepare_dataset.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import json +import os +import random +import shutil +from pathlib import Path + +import torch +from einops import rearrange +from huggingface_hub import snapshot_download + +from cosmos1.models.autoregressive.nemo.utils import read_input_videos +from cosmos1.models.autoregressive.tokenizer.discrete_video import DiscreteVideoFSQJITTokenizer +from cosmos1.models.common.t5_text_encoder import CosmosT5TextEncoder + + +BOV_TOKEN = 64000 +PAD_ID = 64002 + + +def _get_text_prompt_embeddings(prompt, text_encoder): + prompt_embedding, prompt_mask = text_encoder.encode_prompts([prompt]) + return prompt_embedding.squeeze() + + +def _get_video_tokens(filepath, video_tokenizer, data_resolution, num_frames, tokenizer_compression_factor): + input_video = read_input_videos(filepath, data_resolution, num_frames).cuda() + batch_size, channels, frames, height, width = input_video.shape + latent_shape = ( + (frames - 1) // tokenizer_compression_factor[0] + 1, + height // tokenizer_compression_factor[1], + width // tokenizer_compression_factor[2], + ) + T, H, W = latent_shape + video_tokenizer.latent_chunk_duration = T + quantized_out, _ = video_tokenizer.encode(input_video, pixel_chunk_duration=None) + indices = video_tokenizer.fsq_quantizer.codes_to_indices(quantized_out.permute(0, 2, 3, 4, 1)) + indices = rearrange(indices, "B T H W -> (B T H W)") + video_tokens = torch.IntTensor([BOV_TOKEN] + indices.tolist() + [PAD_ID] * 64) + return video_tokens + + +def main(args): + text_encoder = CosmosT5TextEncoder().cuda() + + if args.encoder_path.startswith("nvidia/"): + args.encoder_path = os.path.join(snapshot_download(args.encoder_path), "encoder.jit") + if args.decoder_path.startswith("nvidia/"): + args.decoder_path = os.path.join(snapshot_download(args.decoder_path), "decoder.jit") + + tokenizer_compression_factor = list(map(int, args.tokenizer_compression_factor.split(","))) + assert len(tokenizer_compression_factor) == 3, "Tokenizer compression factor must be a tuple of 3 integers" + + data_resolution = [args.height, args.width] + num_frames = args.num_context_frames + + video_tokenizer = DiscreteVideoFSQJITTokenizer( + enc_fp=args.encoder_path, + dec_fp=args.decoder_path, + name="discrete_video_fsq", + pixel_chunk_duration=num_frames, + ).cuda() + + def save_tensors(jsonl_contents, split): + assert len(jsonl_contents) > 0, ( + f"Ensure length of the {split} split is atleast 1. Modify split string accordingly or add more data points" + ) + for idx, jsonl_content in enumerate(jsonl_contents): + json_data = json.loads(jsonl_content) + assert "prompt" in json_data, "Expected key prompt with text prompt in the input jsonl file" + assert "visual_input" in json_data, ( + "Expected key visual_input with path to video/image in the input jsonl file" + ) + + video_filename = json_data["visual_input"] + prompt = json_data["prompt"] + prompt_embedding = _get_text_prompt_embeddings(prompt, text_encoder) + video_tokens = _get_video_tokens( + video_filename, video_tokenizer, data_resolution, num_frames, tokenizer_compression_factor + ) + + torch.save(prompt_embedding, f"{args.output_dir}/{split}_prompt_{idx}.pt") + torch.save(video_tokens, f"{args.output_dir}/{split}_video_{idx}.pt") + + with open(args.input_jsonl, "r") as f: + jsonl_file_contents = list(f) + + random.shuffle(jsonl_file_contents) + + num_files = len(jsonl_file_contents) + train_split, test_split, val_split = [int(split) for split in args.split_string.split(",")] + assert train_split != 0, "train split in split string is 0. Please make it positive" + assert test_split != 0, "test split in split string is 0. Please make it positive" + assert val_split != 0, "val split in split string is 0. Please make it positive" + total = train_split + test_split + val_split + ( + train_fraction, + test_fraction, + ) = ( + train_split * num_files // total, + test_split * num_files // total, + ) + train_jsonl_file_contents = jsonl_file_contents[:train_fraction] + test_jsonl_file_contents = jsonl_file_contents[train_fraction : (train_fraction + test_fraction)] + val_jsonl_file_contents = jsonl_file_contents[(train_fraction + test_fraction) :] + + if os.path.exists(args.output_dir): + shutil.rmtree(args.output_dir, ignore_errors=True) + + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + save_tensors(train_jsonl_file_contents, "train") + save_tensors(test_jsonl_file_contents, "test") + save_tensors(val_jsonl_file_contents, "val") + + metadata = { + "train_samples": len(train_jsonl_file_contents), + "test_samples": len(test_jsonl_file_contents), + "val_samples": len(val_jsonl_file_contents), + } + + with open(f"{args.output_dir}/metadata.json", "w") as f: + json.dump(metadata, f) + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process some configurations.") + parser.add_argument("--width", default=1024, type=int, help="The width of the video") + parser.add_argument("--height", default=640, type=int, help="The height of the video") + parser.add_argument("--num_context_frames", default=33, type=int, help="The number of frames in the video") + parser.add_argument( + "--tokenizer_compression_factor", default="8,16,16", type=str, help="The compression factor of the tokenizer" + ) + parser.add_argument( + "--input_jsonl", + required=True, + type=str, + help="The path to the a jsonl file. Each line of the file should be a dictionary with two keys. visual_input is a key with the video file as value, and prompt is a key , with the text prompt as value. ", + ) + parser.add_argument( + "--encoder_path", default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", type=str, help="The path to encoder" + ) + parser.add_argument( + "--decoder_path", default="nvidia/Cosmos-1.0-Tokenizer-DV8x16x16", type=str, help="The path to the decoder" + ) + parser.add_argument("--split_string", default="4,1,1", type=str, help="The train/test/val split") + parser.add_argument( + "--output_dir", + required=True, + type=str, + help="The directory to store the prompt embeddings and video tokens", + ) + args = parser.parse_args() + + with torch.no_grad(): + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/utils.py new file mode 100644 index 00000000..45dcfe0c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/nemo/utils.py @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import importlib +import math +import os +from typing import List + +import torch +import torchvision +from huggingface_hub import snapshot_download + +from cosmos1.models.autoregressive.configs.inference.inference_config import DiffusionDecoderSamplingConfig +from cosmos1.models.autoregressive.diffusion_decoder.inference import diffusion_decoder_process_tokens +from cosmos1.models.autoregressive.diffusion_decoder.model import LatentDiffusionDecoderModel +from cosmos1.models.diffusion.inference.inference_utils import ( + load_network_model, + load_tokenizer_model, + skip_init_linear, +) +from cosmos1.utils import log +from cosmos1.utils.config_helper import get_config_module, override + + +DATA_RESOLUTION_DEFAULT = [640, 1024] +NUM_CONTEXT_FRAMES_DEFAULT = 33 + + +def resize_input(video: torch.Tensor, resolution: list[int]): + r""" + Function to perform aspect ratio preserving resizing and center cropping. + This is needed to make the video into target resolution. + Args: + video (torch.Tensor): Input video tensor + resolution (list[int]): Data resolution + Returns: + Cropped video + """ + + orig_h, orig_w = video.shape[2], video.shape[3] + target_h, target_w = resolution + + scaling_ratio = max((target_w / orig_w), (target_h / orig_h)) + resizing_shape = (int(math.ceil(scaling_ratio * orig_h)), int(math.ceil(scaling_ratio * orig_w))) + video_resized = torchvision.transforms.functional.resize(video, resizing_shape) + video_cropped = torchvision.transforms.functional.center_crop(video_resized, resolution) + return video_cropped + + +def read_input_videos( + input_video: str, + data_resolution: list[int] = DATA_RESOLUTION_DEFAULT, + num_frames: int = NUM_CONTEXT_FRAMES_DEFAULT, +) -> torch.tensor: + """Utility to read the input video and return a torch tensor + + Args: + input_video (str): A path to .mp4 file + data_resolution (list, optional): The . Defaults to [640, 1024]. + num_frames (int, optional): The number of frames to read. + + Returns: + A torch tensor of the video + """ + video, _, _ = torchvision.io.read_video(input_video) + video = video.float() / 255.0 + video = video * 2 - 1 + if video.shape[0] > num_frames: + video = video[0:num_frames, :, :, :] + else: + log.info(f"Video doesn't have {num_frames} frames. Padding the video with the last frame.") + # Pad the video + nframes_in_video = video.shape[0] + video = torch.cat( + (video, video[-1, :, :, :].unsqueeze(0).repeat(num_frames - nframes_in_video, 1, 1, 1)), + dim=0, + ) + + video = video[0:num_frames, :, :, :] + video = video.permute(0, 3, 1, 2) + video = resize_input(video, data_resolution) + return video.transpose(0, 1).unsqueeze(0) + + +def run_diffusion_decoder_model(indices_tensor_cur_batch: List[torch.Tensor], out_videos_cur_batch): + """Run a 7b diffusion model to enhance generation output + + Args: + indices_tensor_cur_batch (List[torch.Tensor]): The index tensor(i.e) prompt + generation tokens + out_videos_cur_batch (torch.Tensor): The output decoded video of shape [bs, 3, 33, 640, 1024] + """ + diffusion_decoder_ckpt_path = snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Decoder-DV8x16x16ToCV8x8x8") + dd_tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + tokenizer_corruptor_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-DV8x16x16") + + diffusion_decoder_model = load_model_by_config( + config_job_name="DD_FT_7Bv1_003_002_tokenizer888_spatch2_discrete_cond_on_token", + config_file="cosmos1/models/autoregressive/diffusion_decoder/config/config_latent_diffusion_decoder.py", + model_class=LatentDiffusionDecoderModel, + encoder_path=os.path.join(tokenizer_corruptor_dir, "encoder.jit"), + decoder_path=os.path.join(tokenizer_corruptor_dir, "decoder.jit"), + ) + load_network_model(diffusion_decoder_model, os.path.join(diffusion_decoder_ckpt_path, "model.pt")) + load_tokenizer_model(diffusion_decoder_model, dd_tokenizer_dir) + + generic_prompt = dict() + aux_vars = torch.load(os.path.join(diffusion_decoder_ckpt_path, "aux_vars.pt"), weights_only=True) + generic_prompt["context"] = aux_vars["context"].cuda() + generic_prompt["context_mask"] = aux_vars["context_mask"].cuda() + + output_video = diffusion_decoder_process_tokens( + model=diffusion_decoder_model, + indices_tensor=indices_tensor_cur_batch, + dd_sampling_config=DiffusionDecoderSamplingConfig(), + original_video_example=out_videos_cur_batch[0], + t5_emb_batch=[generic_prompt["context"]], + ) + + del diffusion_decoder_model + gc.collect() + torch.cuda.empty_cache() + + return output_video + + +def load_model_by_config( + config_job_name, + config_file="projects/cosmos_video/config/config.py", + model_class=LatentDiffusionDecoderModel, + encoder_path=None, + decoder_path=None, +): + config_module = get_config_module(config_file) + config = importlib.import_module(config_module).make_config() + + config = override(config, ["--", f"experiment={config_job_name}"]) + + # Check that the config is valid + config.validate() + # Freeze the config so developers don't change it during training. + config.freeze() # type: ignore + if encoder_path: + config.model.tokenizer_corruptor["enc_fp"] = encoder_path + if decoder_path: + config.model.tokenizer_corruptor["dec_fp"] = decoder_path + # Initialize model + with skip_init_linear(): + model = model_class(config.model) + return model diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/networks/transformer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/networks/transformer.py new file mode 100644 index 00000000..65fb4f81 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/networks/transformer.py @@ -0,0 +1,460 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, Dict, Optional + +import torch +import torch.nn as nn +from cosmos1.models.autoregressive.modules.attention import Attention +from cosmos1.models.autoregressive.modules.embedding import ( + RotaryPositionEmbeddingPytorchV1, + RotaryPositionEmbeddingPytorchV2, + SinCosPosEmbAxisTE, +) +from cosmos1.models.autoregressive.modules.mlp import MLP +from cosmos1.models.autoregressive.modules.normalization import create_norm +from cosmos1.models.autoregressive.utils.checkpoint import process_state_dict, substrings_to_ignore +from cosmos1.models.autoregressive.utils.misc import maybe_convert_to_namespace +from cosmos1.utils import log +from torch.nn.modules.module import _IncompatibleKeys + + +class TransformerBlock(nn.Module): + """ + A single transformer block consisting of an attention layer and a feed-forward layer. + """ + + def __init__(self, layer_id: int, args=None): + """ + Initializes the TransformerBlock module. + + Args: + layer_id: The ID of the transformer block. + args: The model arguments containing hyperparameters. + """ + super().__init__() + args = maybe_convert_to_namespace(args) + attention_args = { + "n_heads": args["n_heads"], + "n_kv_heads": args["n_kv_heads"], + "dim": args["dim"], + "context_dim": None, + "max_batch_size": args["max_batch_size"], + "max_seq_len": args["max_seq_len"], + "use_qk_normalization": args["use_qk_normalization"], + "causal_mask": args["causal_mask"], + "head_dim": args["head_dim"], + "fuse_qkv": getattr(args, "fuse_qkv", False), + "precision": getattr(args, "precision", "bfloat16"), + "attn_type": getattr(args, "attn_type", "self"), + } + self.attention = Attention(**attention_args) + + self.has_cross_attention = False + self.cross_attention, self.cross_attention_norm = None, None + + if args["insert_cross_attn"] and layer_id % args["insert_cross_attn_every_k_layers"] == 0: + self.has_cross_attention = True + cross_attention_args = attention_args.copy() + cross_attention_args.update({"context_dim": args["context_dim"], "fuse_qkv": False, "attn_type": "cross"}) + self.cross_attention = Attention(**cross_attention_args) + self.cross_attention_norm = create_norm(args["norm_type"], dim=args["dim"], eps=args["norm_eps"]) + + self.feed_forward = MLP( + dim=args["dim"], + hidden_dim=args["ffn_hidden_size"], + ) + self.layer_id = layer_id + self.attention_norm = create_norm(args["norm_type"], dim=args["dim"], eps=args["norm_eps"]) + self.ffn_norm = create_norm(args["norm_type"], dim=args["dim"], eps=args["norm_eps"]) + + def forward( + self, + x: torch.Tensor, + rope: RotaryPositionEmbeddingPytorchV2, + input_pos: Optional[torch.Tensor] = None, + mask: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Performs the forward pass of the TransformerBlock module. + + Args: + x: The input tensor. + input_pos: The position of the current sequence. Used in inference (with KV cache) only. + freqs_cis: The precomputed frequency values for rotary position embeddings. + mask: The attention mask tensor. + context (Optional[torch.Tensor]): The context tensor added via cross-attn. + context_mask (Optional[torch.Tensor]): The context cross-attn mask tensor. + + Returns: + The output tensor after applying the transformer block. + """ + # Apply attention and residual connection + h = x + self.attention(self.attention_norm(x), rope=rope, input_pos=input_pos, mask=mask) + + # If insert cross-attention, apply CA and residual connection + if self.has_cross_attention: + h = h + self.cross_attention( + self.cross_attention_norm(h), rope=rope, input_pos=input_pos, mask=context_mask, context=context + ) + + # Apply feed-forward network and residual connection + out = h + self.feed_forward(self.ffn_norm(h)) + return out + + def init_weights(self): + """ + Initializes the weights of the transformer block. + """ + for norm in (self.attention_norm, self.ffn_norm): + norm.reset_parameters() + self.attention.init_weights(self.weight_init_std) + self.feed_forward.init_weights(self.weight_init_std) + + if self.has_cross_attention: + self.cross_attention_norm.reset_parameters() + self.cross_attention.init_weights(self.weight_init_std) + # zero-init the final output layer of cross-attention + # nn.init.zeros_(self.cross_attention.wo.weight) + + +class Transformer(nn.Module): + """ + The Transformer network consisting of transformer blocks. + """ + + def __init__(self, params, tokenizer_config=None, init_weights: bool = True): + """ + Initializes the Transformer module. + + Args: + params: The model parameters containing hyperparameters. + tokenizer_config: The model tokenizer configuration. + init_weights (bool): Whether to initialize the weights of the transformer following + TorchTitan's Llama3 initialization scheme. + """ + super().__init__() + # Check if self.params is an OmegaConf DictConfig instance + self.params = maybe_convert_to_namespace(params) + self.vocab_size = params["vocab_size"] + self.n_layers = params["n_layers"] + self.precision = getattr(torch, params["precision"]) + self.tokenizer_config = tokenizer_config + self.num_video_frames = params["num_video_frames"] + + # Token embeddings + self.tok_embeddings = self._create_token_embeddings() + self.rope_config = self._create_rope_config() + + # Transformer layers + self.layers = nn.ModuleList( + [TransformerBlock(layer_id, self.params).to(self.precision) for layer_id in range(self.n_layers)] + ) + + # Final layer normalization + self.norm = create_norm(self.params["norm_type"], dim=self.params["dim"], eps=self.params["norm_eps"]).to( + self.precision + ) + if self.params["pytorch_rope_version"] == "v1": + self.rope = RotaryPositionEmbeddingPytorchV1(**self.rope_config) + elif self.params["pytorch_rope_version"] == "v2": + # Rotary position embeddings + training_type = self.tokenizer_config.training_type if self.tokenizer_config is not None else None + self.rope = RotaryPositionEmbeddingPytorchV2( + seq_len=self.params["max_seq_len"], training_type=training_type, **self.rope_config + ) + else: + raise ValueError(f"Invalid PyTorch RoPE version: {self.params['pytorch_rope_version']}") + # Causal mask + self.causal_mask = torch.tril( + torch.ones(self.params["max_seq_len"], self.params["max_seq_len"], dtype=torch.bool) + ).cuda() + + # Output projection + self.output = self._create_output_projection() + + # Freeze network parameters for finetuning w/ cross-attention + self.has_cross_attention = getattr(params, "insert_cross_attn", False) + + # Absolute position embeddings + if self.params["apply_abs_pos_emb"]: + self.pos_emb_config = self._create_abs_pos_emb_config() + self.pos_emb, self.abs_pos_emb = self._initialize_abs_pos_emb() + + def _create_rope_config(self) -> Dict: + shape_map = { + "3D": self.params["video_latent_shape"], + "1D": None, + } + latent_shape = shape_map.get(self.params["rope_dim"], None) + head_dim = self.params["head_dim"] + if head_dim is None: + head_dim = self.params["dim"] // self.params["n_heads"] + return { + "dim": head_dim, + "max_position_embeddings": self.params["max_seq_len"], + "original_max_position_embeddings": self.params["original_seq_len"], + "rope_theta": self.params["rope_theta"], + "apply_yarn": self.params["apply_yarn"], + "scale": self.params["yarn_scale"], + "beta_fast": self.params["yarn_beta_fast"], + "beta_slow": self.params["yarn_beta_slow"], + "rope_dim": self.params["rope_dim"], + "latent_shape": latent_shape, + "original_latent_shape": self.params["original_latent_shape"], + "pad_to_multiple_of": self.params["pad_to_multiple_of"], + } + + def _create_abs_pos_emb_config(self): + shape_map = { + "3D": self.params["video_latent_shape"], + "1D": None, + } + latent_shape = shape_map.get(self.params["rope_dim"], None) + return { + "dim": self.params["dim"], + "latent_shape": latent_shape, + "pad_to_multiple_of": self.params["pad_to_multiple_of"], + } + + def _create_token_embeddings(self, vocab_size: int = None): + """ + Create token embeddings. + + Returns: + nn.Module: Token embeddings module. + """ + if vocab_size is None: + vocab_size = self.params["vocab_size"] + return nn.Embedding(vocab_size, self.params["dim"]).to(self.precision) + + def _create_output_projection(self, vocab_size: int = None): + """ + Create the output projection layer. + + Args: + vocab_size (int): Vocabulary size (to override the default vocab size). + Returns: + LinearTE: Output projection layer. + """ + if vocab_size is None: + vocab_size = self.params["vocab_size"] + return nn.Linear(self.params["dim"], vocab_size, bias=False).to(self.precision) + + def _initialize_abs_pos_emb(self): + pos_emb = SinCosPosEmbAxisTE(**self.pos_emb_config) + training_type = self.tokenizer_config.training_type if self.tokenizer_config is not None else None + abs_pos_emb = pos_emb.forward(training_type=training_type) + return pos_emb, abs_pos_emb + + def forward( + self, + tokens: Optional[torch.Tensor] = None, + input_pos: Optional[torch.Tensor] = None, + token_embeddings: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + context_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Performs the forward pass of the Transformer module. + + Args: + tokens (torch.Tensor, optional): The input tensor of token IDs. + input_pos (Optional[torch.Tensor]): The position of the current sequence. Used in inference with KV cache. + token_embeddings (torch.Tensor, optional): Precomputed token embeddings. If provided, tokens should be None. + context (Optional[torch.Tensor]): The context tensor added via cross-attn. + context_mask (Optional[torch.Tensor]): The context cross-attn mask tensor. + Returns: + The output tensor after applying the transformer layers. + """ + # Token embeddings + assert tokens is None or token_embeddings is None, ( + "Either tokens or token_embeddings should be provided, not both." + ) + + if token_embeddings is None: + h = self.tok_embeddings(tokens) + else: + h = token_embeddings + + # Create attention mask + mask = self._create_attention_mask(input_pos=input_pos) + + # Prepare layer arguments + layer_kwargs = self._prepare_layer_kwargs( + input_pos=input_pos, + mask=mask, + context=context, + context_mask=context_mask, + ) + + # Apply transformer layers + for layer in self.layers: + if self.params["apply_abs_pos_emb"]: + h = self.apply_abs_pos_emb(h, input_pos=input_pos) + h = layer(h, **layer_kwargs) + + # Apply final layer normalization + h = self.norm(h) + + # Output linear projection + output = self.output(h) + return output + + def _create_attention_mask(self, input_pos: Optional[torch.Tensor]) -> Optional[torch.Tensor]: + """ + Creates an attention mask for the transformer layers. + + Args: + input_pos[torch.Tensor]: The position of input sequence (used for inference only). + + Returns: + Optional[torch.Tensor]: The attention mask, or None for causal mask. + """ + + assert input_pos is not None, "input_pos must be provided for inference" + mask = self.causal_mask[input_pos] + return mask + + def _prepare_layer_kwargs( + self, + input_pos: Optional[torch.Tensor], + mask: Optional[torch.Tensor], + context: Optional[torch.Tensor], + context_mask: Optional[torch.Tensor], + ) -> Dict[str, Any]: + """ + Prepares the keyword arguments for transformer layers. + + Args: + input_pos (Optional[torch.Tensor]): The position of the current sequence. + mask (Optional[torch.Tensor]): The attention mask. + context (Optional[torch.Tensor]): The context tensor added via cross-attn. + context_mask (Optional[torch.Tensor]): The context cross-attn mask tensor. + + Returns: + Dict[str, Any]: A dictionary of keyword arguments for the transformer layers. + """ + if context is not None: + context = context.to(self.precision) + + if isinstance(mask, torch.Tensor) and mask.ndim == 2: + mask = mask[None, None, :, :] + if isinstance(context_mask, torch.Tensor) and context_mask.ndim == 2: + context_mask = context_mask[None, None, :, :] + + layer_kwargs = { + "mask": mask, + "context": context, + "context_mask": context_mask, + } + + layer_kwargs["input_pos"] = input_pos + layer_kwargs["rope"] = self.rope + + return layer_kwargs + + def apply_abs_pos_emb(self, x: torch.Tensor, input_pos: int = None) -> torch.Tensor: + """ + Applies the absolute position embeddings to the input tensor. + """ + abs_pos_emb = self.abs_pos_emb + abs_pos_emb = abs_pos_emb[:, input_pos, :] if input_pos is not None else abs_pos_emb + return x + abs_pos_emb + + @torch.no_grad() + def expand_vocab( + self, new_vocab_size: int, init_method: str = "gaussian", multiple_of=64, expand_output_layer=True + ): + """ + Expands the vocabulary of the model to the new size. + + Args: + new_vocab_size (int): The new vocabulary size. + init_method (str): The initialization method for new embeddings. + Can be "zero" or "gaussian". Default is "gaussian". + multiple_of (int): The new vocabulary size must be a multiple of this value. Defaults to 64 to fully + leverage the power of NVIDIA TensorCore (source 1: https://x.com/karpathy/status/1621578354024677377, + source 2: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc) + expand_output_layer (bool): Whether to also expand the output layer. Defaults to True. + + Returns: + None + """ + if new_vocab_size <= self.vocab_size: + raise ValueError( + f"New vocabulary size ({new_vocab_size}) must be larger than current size ({self.vocab_size})" + ) + if new_vocab_size % multiple_of != 0: + log.debug(f"New vocabulary size must be a multiple of {multiple_of}. Obtained {new_vocab_size}.") + new_vocab_size = (new_vocab_size // multiple_of + 1) * multiple_of + log.debug(f"Rounded vocabulary size to {new_vocab_size}.") + # Resize token embeddings + old_embeddings = self.tok_embeddings + tensor_kwargs = {"device": old_embeddings.weight.device, "dtype": old_embeddings.weight.dtype} + self.tok_embeddings = self._create_token_embeddings(vocab_size=new_vocab_size).to(**tensor_kwargs) + # Initialize new embeddings + if init_method not in ["zero", "gaussian"]: + raise ValueError(f"Unknown initialization method: {init_method}") + # The default initialization of nn.Embedding is Gaussian, so we don't need to do anything + # if init_method == "gaussian". Only if init_method == "zero", we need to zero out the new embeddings. + if init_method == "zero": + self.tok_embeddings.weight.data[self.vocab_size :].zero_() + + # Copy old embeddings + log.debug( + f"old_embeddings: {old_embeddings.weight.data.shape}, new_embeddings: {self.tok_embeddings.weight.data.shape}, vocab_size: {self.vocab_size}" + ) + self.tok_embeddings.weight.data[: self.vocab_size] = old_embeddings.weight.data + # Resize output layer + old_output = self.output + self.output = self._create_output_projection(vocab_size=new_vocab_size if expand_output_layer else None) + + # Initialize new output weights + self.output.weight.data[self.vocab_size :].zero_() + # Copy old output weights + self.output.weight.data[: self.vocab_size] = old_output.weight.data + + # Update vocab size + self.vocab_size = new_vocab_size + log.debug(f"Expanded vocabulary size to {new_vocab_size}") + + def state_dict(self, *args, **kwargs): + """ + Process the state dict (e.g., remove "_extra_state" keys imposed by TransformerEngine for FP8). + """ + state_dict = super().state_dict(*args, **kwargs) + return process_state_dict(state_dict) + + def load_state_dict(self, state_dict: Dict[str, Any], strict: bool = True, assign: bool = False): + """ + Ignore the missing keys with substrings matching `substring_to_ignore` (e.g., "_extra_state" keys imposed by + TransformerEngine for FP8). + """ + state_dict = process_state_dict(state_dict) + missing_keys, unexpected_keys = super().load_state_dict(state_dict, strict=False, assign=assign) + if strict: + actual_missing_keys = [] + for key in missing_keys: + if not any(substring in key for substring in substrings_to_ignore): + actual_missing_keys.append(key) + if len(actual_missing_keys) > 0 or len(unexpected_keys) > 0: + raise ValueError(f"Missing keys: {actual_missing_keys}\n\nUnexpected keys: {unexpected_keys}") + missing_keys = actual_missing_keys + return _IncompatibleKeys(missing_keys, unexpected_keys) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/networks/vit.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/networks/vit.py new file mode 100644 index 00000000..e8ec1b10 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/networks/vit.py @@ -0,0 +1,412 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +This module implements a Vision Transformer (ViT) with 2D Rotary Position Embeddings, +designed for processing image inputs in vision-language models. + +This module follows Mistral's vision encoder implementation (for their Pistral-12B VLM): +https://github.com/mistralai/mistral-inference/blob/main/src/mistral_inference/vision_encoder.py +""" + +from functools import partial +from typing import Any, Callable, Mapping, Optional, Tuple + +import torch +import torch.nn as nn +from cosmos1.models.autoregressive.modules.normalization import create_norm +from cosmos1.models.autoregressive.networks.transformer import TransformerBlock +from cosmos1.utils import log + + +def get_vit_config(model_name: str) -> Mapping[str, Any]: + """ + Get the ViT configuration for a given model name. + """ + if model_name == "pixtral-12b-vit": + # The 400M ViT of Pixtral 12B VLM + return dict( + dim=1024, + num_channels=3, + image_size=1024, + patch_size=16, + rope_theta=10000, + ffn_hidden_size=4096, + n_layers=24, + n_heads=16, + n_kv_heads=16, + norm_type="rmsnorm", + norm_eps=1e-5, + image_token_id=10, + ) + else: + raise ValueError(f"Unknown model name: {model_name}") + + +def precompute_freqs_cis_2d( + dim: int, + height: int, + width: int, + theta: float, +) -> torch.Tensor: + """ + Precompute 2D complex tensor for rotary position embedding. + + This function generates a 2D complex tensor used for rotary position embeddings, + which helps the model understand spatial relationships in the input image. + + Args: + dim (int): Dimension of the model (typically the hidden size divided by number of heads). + height (int): Height of the image in patches. + width (int): Width of the image in patches. + theta (float): Base value for the angle calculation, controls the frequency range. + + Returns: + torch.Tensor: 2D complex tensor of shape (height, width, dim // 2). + """ + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim)) + + h = torch.arange(height, device=freqs.device) + w = torch.arange(width, device=freqs.device) + + freqs_h = torch.outer(h, freqs[::2]).float() + freqs_w = torch.outer(w, freqs[1::2]).float() + freqs_2d = torch.cat( + [ + freqs_h[:, None, :].repeat(1, width, 1), + freqs_w[None, :, :].repeat(height, 1, 1), + ], + dim=-1, + ) + return torch.polar(torch.ones_like(freqs_2d), freqs_2d) + + +def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor): + """ + Reshape frequency tensor for broadcasting with input tensor. + + This function ensures that the frequency tensor can be properly broadcast + with the input tensor during the rotary embedding process. + + Args: + freqs_cis (torch.Tensor): Frequency tensor from precompute_freqs_cis_2d. + x (torch.Tensor): Input tensor to be embedded. + + Returns: + torch.Tensor: Reshaped frequency tensor ready for broadcasting. + """ + ndim = x.ndim + assert 0 <= 1 < ndim, f"ndim is {ndim} but index is {1}" + assert freqs_cis.shape == ( + x.shape[1], + x.shape[-1], + ), f"freqs_cis shape is {freqs_cis.shape} but x shape is {x.shape}" + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis.view(*shape) + + +def apply_rotary_emb( + xq: torch.Tensor, + xk: torch.Tensor, + *args, + freqs_cis: torch.Tensor, + **kwargs, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary positional embeddings to input tensors. + + This function applies the rotary positional embeddings to the query and key tensors, + which helps the model understand spatial relationships in the input. + + Args: + xq (torch.Tensor): Query tensor. + xk (torch.Tensor): Key tensor. + freqs_cis (torch.Tensor): Precomputed frequencies from precompute_freqs_cis_2d. + *args: Variable length argument list (unused). + **kwargs: Arbitrary keyword arguments (unused). + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Rotated query and key tensors. + """ + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) + freqs_cis = reshape_for_broadcast(freqs_cis, xq_) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) + return xq_out.type_as(xq), xk_out.type_as(xk) + + +class VisionTransformer(nn.Module): + """ + Vision Transformer model for image processing. + + This class implements a Vision Transformer that processes images using a patch-based approach + and applies transformer layers with rotary position embeddings. + + Args: + dim (int): Dimension of the model (hidden size). + num_channels (int): Number of input image channels (e.g., 3 for RGB). + patch_size (int): Size of each image patch (e.g., 16x16 pixels). + n_layers (int): Number of transformer layers. + n_heads (int): Number of attention heads. + ffn_hidden_size (int): Hidden size of the feed-forward network in transformer blocks. + norm_type (str): Type of normalization to use (e.g., "rmsnorm"). + norm_eps (float): Epsilon value for normalization layers. + image_size (int): Size of the input image (assumed square). + rope_theta (float): Base value for rotary position embedding calculation. + attention_dropout (float): Dropout rate for attention layers. + hidden_dropout (float): Dropout rate for hidden layers. + image_token_id (int): Token ID for the image token (if present). + """ + + def __init__( + self, + dim: int = 1024, + num_channels: int = 3, + patch_size: int = 16, + n_layers: int = 24, + n_heads: int = 16, + n_kv_heads: int = None, + ffn_hidden_size: int = 4096, + norm_type: str = "rmsnorm", + norm_eps: float = 1e-5, + image_size: int = 1024, + rope_theta: float = 1000000.0, + image_token_id: int = None, + ): + super().__init__() + self.patch_conv = nn.Conv2d( + in_channels=num_channels, + out_channels=dim, + kernel_size=patch_size, + stride=patch_size, + bias=False, + ) + self.ln_pre = create_norm(norm_type=norm_type, dim=dim, eps=norm_eps) + if n_kv_heads is None: + n_kv_heads = n_heads + layer_args = dict( + n_layers=n_layers, + n_heads=n_heads, + n_kv_heads=n_kv_heads, + dim=dim, + use_qk_normalization=False, + max_seq_len=None, + max_batch_size=None, + ffn_hidden_size=ffn_hidden_size, + norm_type=norm_type, + norm_eps=norm_eps, + causal_mask=False, # Full attention in ViT + head_dim=None, + insert_cross_attn=False, + attn_type="full", + ) + + self.transformer = VisionTransformerBlocks(n_layers=n_layers, args=layer_args) + + head_dim = dim // n_heads + assert head_dim % 2 == 0, "ROPE requires even head_dim" + + self.dim = dim + self.n_heads = n_heads + self.max_patches_per_side = image_size // patch_size + self.image_size = image_size + self.patch_size = patch_size + self.rope_theta = rope_theta + self._freqs_cis: Optional[torch.Tensor] = None + self.image_token_id = image_token_id + + num_params = self.get_num_params() + log.debug(f"Number of model parameters: {round(num_params / 1e6, 3)}M") + + @classmethod + def build( + cls, + config: Mapping[str, Any], + ) -> "VisionTransformer": + """ + Create a Vision Transformer from a configuration dictionary. + + This class method creates a Vision Transformer from a configuration dictionary, + which is typically loaded from a JSON file or other configuration source. + + Args: + config (Mapping[str, Any]): Configuration dictionary for the Vision Transformer. + + Returns: + VisionTransformer: Vision Transformer model instance. + """ + necessary_keys = ["dim", "num_channels", "patch_size", "n_layers", "n_heads", "ffn_hidden_size", "rope_theta"] + missing_keys = [k for k in necessary_keys if k not in config] + assert len(missing_keys) == 0, f"Missing keys in config: {missing_keys}" + return cls( + **config, + ) + + def expand_in_channels(self, new_in_channels: int): + """ + Expand the input channels of the patch convolution layer. + This is useful when the input is non-standard, e.g. a 4-channel image with the last channel as the alpha channel. + Note that you should only call this method after the weight is loaded. + """ + assert new_in_channels > self.patch_conv.in_channels, ( + "Cannot expand the input channels of the patch convolution layer to be less than the original number of channels." + ) + log.debug( + f"Vision encoder in_channels is {self.patch_conv.in_channels}. But you have specified to be {new_in_channels}. We will change it to {new_in_channels} channels with {new_in_channels - self.patch_conv.in_channels} channels of 0s." + ) + new_conv = nn.Conv2d( + in_channels=new_in_channels, + out_channels=self.patch_conv.out_channels, + kernel_size=self.patch_conv.kernel_size, + stride=self.patch_conv.stride, + bias=False, + ) + new_conv.weight.data[:, : self.patch_conv.in_channels].copy_(self.patch_conv.weight.data) + new_conv.weight.data[ + :, self.patch_conv.in_channels : + ].zero_() # zeroize, such that initially it has no effect to output + self.patch_conv = new_conv + + @property + def device(self) -> torch.device: + """Get the device of the model.""" + return next(self.parameters()).device + + @property + def freqs_cis(self) -> torch.Tensor: + """ + Get or compute the frequency tensor for rotary position embedding. + + This property lazily initializes and caches the frequency tensor used for + rotary position embeddings, ensuring it's on the correct device. + + Returns: + torch.Tensor: The frequency tensor for rotary position embeddings. + """ + if self._freqs_cis is None: + self._freqs_cis = precompute_freqs_cis_2d( + dim=self.dim // self.n_heads, + height=self.max_patches_per_side, + width=self.max_patches_per_side, + theta=self.rope_theta, + ) + + if self._freqs_cis.device != self.device: + self._freqs_cis = self._freqs_cis.to(device=self.device) + + return self._freqs_cis + + def forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + """ + Forward pass of the Vision Transformer. + + This method processes the input image through the Vision Transformer, + including patch embedding, position embedding, and transformer layers. + + Args: + x (torch.Tensor): Input tensor of shape (B, C, H, W), where B is batch size, + C is number of channels, and H, W are height and width. + + Returns: + torch.Tensor: Output features of shape (B, N, D), where N is the number of patches + and D is the embedding dimension. + """ + + patch_embeds = self.patch_conv(x) # (B, D, Hp, Wp) + _, _, Hp, Wp = patch_embeds.shape # Patch embeds dim + patch_embeds = patch_embeds.flatten(2) # (B, D, Hp*Wp) + patch_embeds = patch_embeds.transpose(1, 2) # (B, Hp*Wp, D) + patch_embeds = self.ln_pre(patch_embeds) # (B, Hp*Wp, D) + positions = torch.stack( + torch.meshgrid( + torch.arange(Hp), + torch.arange(Wp), + indexing="ij", + ), + dim=-1, + ).reshape(-1, 2) + + freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]] + rope = partial(apply_rotary_emb, freqs_cis=freqs_cis) + out = self.transformer(patch_embeds, rope=rope) + + return out + + def get_num_params( + self, + ) -> int: + """ + Return the number of parameters in the model. + """ + n_params = sum(p.numel() for p in self.parameters()) + return n_params + + +class VisionTransformerBlocks(nn.Module): + """ + Vision Transformer Blocks. + + This class implements a stack of Transformer blocks used in the Vision Transformer. + + Args: + n_layers (int): Number of transformer layers. + args (Mapping[str, Any]): Arguments for each transformer block, including dimensions, + """ + + def __init__( + self, + n_layers: int, + args: Mapping[str, Any], + ): + super().__init__() + self.layers = torch.nn.ModuleList() + + for layer_id in range(n_layers): + self.layers.append( + TransformerBlock( + layer_id=layer_id, + args=args, + ) + ) + + def forward( + self, + x: torch.Tensor, + rope: Callable, + ) -> torch.Tensor: + """ + Forward pass through the Vision Transformer Blocks. + + This method applies a series of Transformer blocks to the input tensor, + using the provided rotary position embedding function. + + Args: + x (torch.Tensor): Input tensor of shape (B, N, D), where B is batch size, + N is the number of patches, and D is the embedding dimension. + rope (Callable): Rotary position embedding function to be applied in each layer. + + Returns: + torch.Tensor: Output tensor after passing through all transformer layers, + with the same shape as the input. + """ + for layer in self.layers: + x = layer(x, input_pos=None, mask=None, rope=rope) + return x diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/discrete_video.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/discrete_video.py new file mode 100644 index 00000000..932c3e33 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/discrete_video.py @@ -0,0 +1,363 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import torch +from einops import rearrange + +from cosmos1.models.autoregressive.tokenizer.quantizers import FSQuantizer + + +# Make sure jit model output consistenly during consecutive calls +# Check here: https://github.com/pytorch/pytorch/issues/74534 +torch._C._jit_set_texpr_fuser_enabled(False) + + +def load_jit_model(jit_filepath: str = None, device: str = "cuda") -> torch.jit.ScriptModule: + """Loads a torch.jit.ScriptModule from a filepath. + + Args: + jit_filepath: The filepath to the JIT-compiled model. + device: The device to load the model onto, default=cuda. + Returns: + The JIT compiled model loaded to device and on eval mode. + """ + # Make sure jit model output consistenly during consecutive calls + # Check here: https://github.com/pytorch/pytorch/issues/74534 + torch._C._jit_set_texpr_fuser_enabled(False) + + model = torch.jit.load(jit_filepath) + return model.eval().to(device) + + +class BaseDiscreteVideoFSQTokenizer(torch.nn.Module): + """ + A base class for Discrete Video FSQ Tokenizer that handles data type conversions, and normalization + using provided mean and standard deviation values for latent space representation. + Derived classes should load pre-trained encoder and decoder components into a encoder and decoder attributes. + + Attributes: + encoder (Module | Callable): Encoder loaded from storage. + decoder (Module | Callable): Decoder loaded from storage. + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + name (str): Name of the model, used for differentiating cache file paths. + latent_ch (int, optional): Number of latent channels (default is 6). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + pixel_chunk_duration (int): The duration (in number of frames) of each chunk of video data at the pixel level. + latent_chunk_duration (int): The duration (in number of frames) of each chunk at the latent representation level. + max_enc_batch_size (int): The maximum batch size to process in one go to avoid memory overflow. + level (list[int]): The level defined in FSQ quantizer. + compression_ratio (list[int]): The compression factor for (T, H, W). + """ + + def __init__( + self, + name: str, + latent_ch: int = 6, + is_bf16: bool = True, + pixel_chunk_duration: int = 25, + latent_chunk_duration: int = 4, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + levels: list[int] = [8, 8, 8, 5, 5, 5], + compression_ratio: list[int] = [8, 16, 16], + ): + super().__init__() + self.channel = latent_ch + self.name = name + dtype = torch.bfloat16 if is_bf16 else torch.float32 + self.dtype = dtype + self.pixel_chunk_duration = pixel_chunk_duration + self.latent_chunk_duration = latent_chunk_duration + self.max_enc_batch_size = max_enc_batch_size + self.max_dec_batch_size = max_dec_batch_size + self.levels = levels + self.compress_ratio = compression_ratio + self.fsq_quantizer = FSQuantizer(levels) + + @property + def latent_ch(self) -> int: + """ + Returns the number of latent channels in the tokenizer. + """ + return self.channel + + @torch.no_grad() + def encode(self, state: torch.Tensor, pixel_chunk_duration: Optional[int] = None) -> torch.Tensor: + B, C, T, H, W = state.shape + if pixel_chunk_duration is None: + # Use the default pixel chunk duration and latent chunk duration + pixel_chunk_duration = self.pixel_chunk_duration + latent_chunk_duration = self.latent_chunk_duration + else: + # Update the latent chunk duration based on the given pixel chunk duration + latent_chunk_duration = 1 + (pixel_chunk_duration - 1) // self.compress_ratio[0] + + assert T % pixel_chunk_duration == 0, ( + f"Temporal dimension {T} is not divisible by chunk_length {pixel_chunk_duration}" + ) + state = rearrange(state, "b c (n t) h w -> (b n) c t h w", t=pixel_chunk_duration) + + # use max_enc_batch_size to avoid OOM + if state.shape[0] > self.max_enc_batch_size: + quantized_out_list = [] + indices_list = [] + for i in range(0, state.shape[0], self.max_enc_batch_size): + indices, quantized_out, _ = self.encoder(state[i : i + self.max_enc_batch_size].to(self.dtype)) + quantized_out_list.append(quantized_out) + indices_list.append(indices) + quantized_out = torch.cat(quantized_out_list, dim=0) + indices = torch.cat(indices_list, dim=0) + else: + indices, quantized_out, _ = self.encoder(state.to(self.dtype)) + assert quantized_out.shape[2] == latent_chunk_duration + return rearrange(quantized_out, "(b n) c t h w -> b c (n t) h w", b=B), rearrange( + indices, "(b n) t h w -> b (n t) h w", b=B + ) + + @torch.no_grad() + def decode(self, indices: torch.Tensor, pixel_chunk_duration: Optional[int] = None) -> torch.Tensor: + B, T, _, _ = indices.shape + if pixel_chunk_duration is None: + pixel_chunk_duration = self.pixel_chunk_duration + latent_chunk_duration = self.latent_chunk_duration + else: + latent_chunk_duration = 1 + (pixel_chunk_duration - 1) // self.compress_ratio[0] + assert T % latent_chunk_duration == 0, ( + f"Temporal dimension {T} is not divisible by chunk_length {latent_chunk_duration}" + ) + indices = rearrange(indices, "b (n t) h w -> (b n) t h w", t=latent_chunk_duration) + + # use max_dec_batch_size to avoid OOM + if indices.shape[0] > self.max_dec_batch_size: + state = [] + for i in range(0, indices.shape[0], self.max_dec_batch_size): + state.append(self.decoder(indices[i : i + self.max_dec_batch_size])) + state = torch.cat(state, dim=0) + else: + state = self.decoder(indices) + + assert state.shape[2] == pixel_chunk_duration + return rearrange(state, "(b n) c t h w -> b c (n t) h w", b=B) + + def reset_dtype(self, *args, **kwargs): + """ + Resets the data type of the encoder and decoder to the model's default data type. + + Args: + *args, **kwargs: Unused, present to allow flexibility in method calls. + """ + del args, kwargs + self.decoder.to(self.dtype) + self.encoder.to(self.dtype) + + +class DiscreteVideoFSQJITTokenizer(BaseDiscreteVideoFSQTokenizer): + """ + A JIT compiled Discrete Video FSQ Tokenizer that loads pre-trained encoder + and decoder components from a remote store, handles data type conversions, and normalization + using provided mean and standard deviation values for latent space representation. + + Attributes: + encoder (Module): The JIT compiled encoder loaded from storage. + decoder (Module): The JIT compiled decoder loaded from storage. + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + enc_fp (str): File path to the encoder's JIT file on the remote store. + dec_fp (str): File path to the decoder's JIT file on the remote store. + name (str): Name of the model, used for differentiating cache file paths. + latent_ch (int, optional): Number of latent channels (default is 6). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + pixel_chunk_duration (int): The duration (in number of frames) of each chunk of video data at the pixel level. + latent_chunk_duration (int): The duration (in number of frames) of each chunk at the latent representation level. + max_enc_batch_size (int): The maximum batch size to process in one go to avoid memory overflow. + level (list[int]): The level defined in FSQ quantizer. + compression_ratio (list[int]): The compression factor for (T, H, W). + """ + + def __init__( + self, + enc_fp: str, + dec_fp: str, + name: str, + latent_ch: int = 6, + is_bf16: bool = True, + pixel_chunk_duration: int = 25, + latent_chunk_duration: int = 4, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + levels: list[int] = [8, 8, 8, 5, 5, 5], + compression_ratio: list[int] = [8, 16, 16], + ): + super().__init__( + name, + latent_ch, + is_bf16, + pixel_chunk_duration, + latent_chunk_duration, + max_enc_batch_size, + max_dec_batch_size, + levels, + compression_ratio, + ) + + self.load_encoder(enc_fp) + self.load_decoder(dec_fp) + + def load_encoder(self, enc_fp: str) -> None: + """ + Load the encoder from the remote store. + + Args: + - enc_fp (str): File path to the encoder's JIT file on the remote store. + """ + self.encoder = load_jit_model(enc_fp, device="cuda") + self.encoder.eval() + for param in self.encoder.parameters(): + param.requires_grad = False + self.encoder.to(self.dtype) + + def load_decoder(self, dec_fp: str) -> None: + """ + Load the decoder from the remote store. + + Args: + - dec_fp (str): File path to the decoder's JIT file on the remote store. + """ + self.decoder = load_jit_model(dec_fp, device="cuda") + self.decoder.eval() + for param in self.decoder.parameters(): + param.requires_grad = False + self.decoder.to(self.dtype) + + +class DiscreteVideoFSQStateDictTokenizer(BaseDiscreteVideoFSQTokenizer): + """ + A Discrete Video FSQ Tokenizer that loads weights from pre-trained JITed encoder + into as nn.Module so that encoder can be "torch.compile()" and JITed decoder, so it can be torch.compiled, + handles data type conversions, and normalization using provided mean and standard deviation values for latent + space representation. + + Attributes: + tokenizer_module (Module): Tokenizer module with weights loaded from JIT checkpoints + encoder (Callable): tokenizer_module's encode method + decoder (Callable): tokenizer_module's decode method + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + enc_fp (str): File path to the encoder's JIT file on the remote store. + dec_fp (str): File path to the decoder's JIT file on the remote store. + tokenizer_module (Module): Tokenizer module that will have it's weights loaded + name (str): Name of the model, used for differentiating cache file paths. + latent_ch (int, optional): Number of latent channels (default is 6). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + pixel_chunk_duration (int): The duration (in number of frames) of each chunk of video data at the pixel level. + latent_chunk_duration (int): The duration (in number of frames) of each chunk at the latent representation level. + max_enc_batch_size (int): The maximum batch size to process in one go to avoid memory overflow. + level (list[int]): The level defined in FSQ quantizer. + compression_ratio (list[int]): The compression factor for (T, H, W). + """ + + def __init__( + self, + enc_fp: str, + dec_fp: str, + tokenizer_module: torch.nn.Module, + name: str, + latent_ch: int = 6, + is_bf16: bool = True, + pixel_chunk_duration: int = 25, + latent_chunk_duration: int = 4, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + levels: list[int] = [8, 8, 8, 5, 5, 5], + compression_ratio: list[int] = [8, 16, 16], + ): + super().__init__( + name, + latent_ch, + is_bf16, + pixel_chunk_duration, + latent_chunk_duration, + max_enc_batch_size, + max_dec_batch_size, + levels, + compression_ratio, + ) + + self.load_encoder_and_decoder(enc_fp, dec_fp, tokenizer_module) + + def load_encoder_and_decoder(self, enc_fp: str, dec_fp: str, tokenizer_module: torch.nn.Module) -> None: + """ + Load the encoder from the remote store. + + Args: + - enc_fp (str): File path to the encoder's JIT file on the remote store. + - def_fp (str): File path to the decoder's JIT file on the remote store. + - tokenizer_module (Module): Tokenizer module that was used to create JIT checkpoints + """ + self.decoder = load_jit_model(dec_fp) + + self.decoder.eval() + for param in self.decoder.parameters(): + param.requires_grad = False + self.decoder.to(self.dtype) + + encoder_sd = load_jit_model(enc_fp).state_dict() + + del tokenizer_module.post_quant_conv + del tokenizer_module.decoder + + state_dict = { + k: v + for k, v in (encoder_sd).items() + # Variables captured by JIT + if k + not in ( + "encoder.patcher3d.wavelets", + "encoder.patcher3d._arange", + "encoder.patcher3d.patch_size_buffer", + "quantizer._levels", + "quantizer._basis", + "quantizer.implicit_codebook", + ) + } + + tokenizer_module.load_state_dict(state_dict) + + tokenizer_module.eval() + for param in tokenizer_module.parameters(): + param.requires_grad = False + tokenizer_module.to(self.dtype) + + self.tokenizer_module = tokenizer_module + self.encoder = self.tokenizer_module.encode + + def reset_dtype(self, *args, **kwargs): + """ + Resets the data type of the encoder and decoder to the model's default data type. + + Args: + *args, **kwargs: Unused, present to allow flexibility in method calls. + """ + del args, kwargs + self.decoder.to(self.dtype) + self.tokenizer_module.to(self.dtype) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/image_text_tokenizer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/image_text_tokenizer.py new file mode 100644 index 00000000..e0dc7d48 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/image_text_tokenizer.py @@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, Dict, List, Optional, Union + +import numpy as np +import PIL +import torch +import transformers +from transformers import AutoImageProcessor +from transformers.image_utils import ImageInput, is_valid_image, load_image + +from cosmos1.models.autoregressive.tokenizer.text_tokenizer import TextTokenizer +from cosmos1.utils import log + + +# Configuration for different vision-language models +IMAGE_CONFIGS = { + "pixtral": { + "patch_size": 16, + "image_token": "[IMG]", + "image_break_token": "[IMG_BREAK]", + "image_end_token": "[IMG_END]", + } +} + +# Chat template for Pixtral-12B-Instruct +PIXTRAL_CHAT_TEMPLATE = '{%- if messages[0]["role"] == "system" %}\n {%- set system_message = messages[0]["content"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if (message[\'role\'] == \'user\') != (loop.index0 % 2 == 0) %}\n {{- raise_exception(\'After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\') }}\n {%- endif %}\n {%- if message["role"] == "user" %}\n {%- if loop.last and system_message is defined %}\n {{- "[INST]" + system_message + "\n\n" }}\n {%- else %}\n {{- "[INST]" }}\n {%- endif %}\n {%- if message["content"] is not string %}\n {%- for chunk in message["content"] %}\n {%- if chunk["type"] == "text" %}\n {{- chunk["content"] }}\n {%- elif chunk["type"] == "image" %}\n {{- "[IMG]" }}\n {%- else %}\n {{- raise_exception("Unrecognized content type!") }}\n {%- endif %}\n {%- endfor %}\n {%- else %}\n {{- message["content"] }}\n {%- endif %}\n {{- "[/INST]" }}\n {%- elif message["role"] == "assistant" %}\n {{- message["content"] + eos_token}}\n {%- else %}\n {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}\n {%- endif %}\n{%- endfor %}' + + +# Copied from transformers.models.pixtral.processing_pixtral.is_url +def is_url(val) -> bool: + """Check if the given value is a URL.""" + return isinstance(val, str) and val.startswith("http") + + +# Copied from transformers.models.pixtral.processing_pixtral.is_image_or_image_url +def is_image_or_image_url(elem): + """Check if the given element is an image or an image URL.""" + return is_url(elem) or is_valid_image(elem) + + +def load_image_list( + image_list: List[Union[str, "PIL.Image.Image"]], timeout: Optional[float] = None +) -> List["PIL.Image.Image"]: + """ + Load a list of images. + + Args: + image_list (List[Union[str, PIL.Image.Image]]): The list of images to load. + timeout (Optional[float]): The timeout for loading the image. + + Returns: + List[PIL.Image.Image]: The list of loaded images. + """ + return [load_image(image, timeout=timeout) for image in image_list] + + +class ImageTextTokenizer(TextTokenizer): + """ + Image-text tokenizer class that extends the text tokenizer to support vision tokens as well. + """ + + def __init__( + self, + model_family: str, + is_instruct_model: bool, + tokenizer_path: str, + image_processor_path: str, + ): + """ + Initialize the ImageTextTokenizer. + + Args: + model_family (str): The model family. + is_instruct_model (bool): Whether the model is an instruct model. + s3_credential_path (str): The path to the s3 credential file. Defaults to "credentials/pbss_dir.secret". + + Raises: + AssertionError: If the model family is not supported or if the transformers version is incompatible. + """ + super().__init__( + model_family=model_family, + is_instruct_model=is_instruct_model, + local_path=tokenizer_path, + ) + assert model_family in ["pixtral"], f"Unsupported model family: {model_family}" + if model_family == "pixtral": + # Need transformers>=4.45.0 + assert transformers.__version__ >= "4.45.0", "Pixtral requires transformers>=4.45.0" + assert is_instruct_model, "Pixtral requires is_instruct_model=True" + if not hasattr(self.tokenizer, "chat_template") or self.tokenizer.chat_template is None: + setattr(self.tokenizer, "chat_template", PIXTRAL_CHAT_TEMPLATE) + log.debug(f"Pixtral tokenizer chat template set to: {PIXTRAL_CHAT_TEMPLATE}") + + # Set up image-specific configurations + image_config = IMAGE_CONFIGS[model_family] + self.patch_size = image_config["patch_size"] + self.image_token = image_config["image_token"] + self.image_break_token = image_config["image_break_token"] + self.image_end_token = image_config["image_end_token"] + + # Initialize the image processor + self.image_processor = AutoImageProcessor.from_pretrained(image_processor_path) + + def encode( + self, + text: Union[str, List[str], List[int]], + *, # Enforce keyword-only arguments + images: Optional[ImageInput] = None, + image_kwargs: Optional[Dict[str, Any]] = None, + **text_kwargs, + ) -> List[int]: + """ + Process the images and return the tokenized images and text. + + Args: + text (`str`, `List[str]`, `List[List[str]]`): + The sequence or batch of sequences to be encoded. + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of images to be prepared. + image_kwargs (Optional[Dict[str, Any]]): Additional keyword arguments for image processing. + **text_kwargs: Additional keyword arguments for text processing. + + Returns: + A dictionary with the following fields: + - **input_ids** -- List of token ids to be fed to a model. + - **attention_mask** -- List of indices specifying which tokens should be attended to by the model. + - **pixel_values** -- Pixel values to be fed to a model. + + Raises: + ValueError: If the input images are in an invalid format. + """ + + output_dict, image_inputs = {}, {} + if images is not None: + # Preprocess images + if is_image_or_image_url(images): + images = [[images]] + elif isinstance(images, list) and is_image_or_image_url(images[0]): + images = [images] + elif ( + not isinstance(images, list) + and not isinstance(images[0], list) + and not is_image_or_image_url(images[0][0]) + ): + raise ValueError( + "Invalid input images. Please provide a single image or a list of images or a list of list of images." + ) + + # Load and process images + images = [load_image_list(sample) for sample in images] + image_kwargs = image_kwargs or {} + image_inputs = self.image_processor( + images, patch_size=self.patch_size, return_tensors="np", **image_kwargs + ) + + # Validate image inputs + assert "pixel_values" in image_inputs, "pixel_values not found in image_inputs" + assert "image_sizes" in image_inputs, "image_sizes not found in image_inputs" + assert len(image_inputs.keys()) == 2, "Only one key is allowed in image_inputs, got {}".format( + image_inputs.keys() + ) + + # Extract pixel values and image sizes + pixel_values = image_inputs["pixel_values"][0] + image_sizes = image_inputs["image_sizes"][0] + unique_sizes = np.unique(image_sizes, axis=0) + + assert len(unique_sizes) == 1, "All images must have the same size, got {}".format(unique_sizes) + + # Convert pixel values to PyTorch tensor + pixel_values = np.asarray(pixel_values) + pixel_values = torch.from_numpy(pixel_values) + output_dict["pixel_values"] = pixel_values + output_dict["image_sizes"] = image_sizes + + # Expand image tokens in text + if image_inputs.get("pixel_values") is not None: + replace_strings = [] + # Calculate the number of tokens needed for each image and create a placeholder + for image_size in image_sizes: + height, width = image_size + num_height_tokens = height // self.patch_size + num_width_tokens = width // self.patch_size + replace_tokens = [[self.image_token] * num_width_tokens + [self.image_break_token]] * num_height_tokens + # Flatten list + replace_tokens = [item for sublist in replace_tokens for item in sublist] + replace_tokens[-1] = self.image_end_token + replace_str = "".join(replace_tokens) + replace_strings.append(replace_str) + text = text.replace(self.image_token, "", 1) + + # Replace placeholders with actual image token sequences + while "" in text: + replace_str = replace_strings.pop(0) + text = text.replace("", replace_str, 1) + + # Encode the text + text_inputs = super(ImageTextTokenizer, self).encode(text, **text_kwargs) + + output_dict["input_ids"] = text_inputs + return output_dict + + def apply_chat_template( + self, + conversation: List[Dict[str, Any]] | List[List[Dict[str, Any]]], + *, + images: Optional[ImageInput] = None, + image_kwargs: Optional[Dict[str, Any]] = None, + add_generation_prompt: bool = False, + tokenize: bool = True, + padding: bool = False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors: Optional[str] = None, + return_dict: bool = True, + return_assistant_tokens_mask: bool = False, + generation_prefix: str = "", + tokenizer_kwargs: Optional[Dict[str, Any]] = None, + **kwargs, + ): + """ + Apply the chat template to the conversation. + + Args: + conversation (List[Dict[str, Any]] | List[List[Dict[str, Any]]]): The conversation to process. + images (Optional[ImageInput]): Images to include in the conversation. + image_kwargs (Optional[Dict[str, Any]]): Additional keyword arguments for image processing. + add_generation_prompt (bool): Whether to add a generation prompt. + tokenize (bool): Whether to tokenize the output. + padding (bool): Whether to pad the output. + truncation (bool): Whether to truncate the output. + max_length (Optional[int]): Maximum length of the output. + return_tensors (Optional[str]): The type of tensors to return. + return_dict (bool): Whether to return a dictionary. + return_assistant_tokens_mask (bool): Whether to return the assistant tokens mask. + generation_prefix (str): Prefix to add before asking model to generate. Helpful to guide the generation. Defaults to "". + tokenizer_kwargs (Optional[Dict[str, Any]]): Additional keyword arguments for the tokenizer. + **kwargs: Additional keyword arguments. + + Returns: + The processed conversation with applied chat template. + + Raises: + AssertionError: If return_dict is False or if the conversation format is invalid. + """ + assert return_dict, "return_dict must be True for ImageTextTokenizer" + assert isinstance(conversation, list), "conversation must be a list" + if isinstance(conversation[0], list): + assert len(conversation) == 1, "Only support single-conversation input, got {}".format(conversation) + conversation = conversation[0] + + # Extract images from the conversation if not provided + if images is None: + images = [] + for msg in conversation: + if msg.get("images", None) is not None: + images = images + (msg["images"]) + images = load_image_list(images) + # In case the input does not have images, will ignore + # Useful in feeding VLM inputs with and without images + if isinstance(images, list) and len(images) == 0: + images = None + + # Apply the chat template to the text + text = super().apply_chat_template( + conversation, + tokenize=False, + add_generation_prompt=add_generation_prompt, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_dict=False, + return_assistant_tokens_mask=return_assistant_tokens_mask, + generation_prefix=generation_prefix, + tokenizer_kwargs=tokenizer_kwargs, + **kwargs, + ) + + if tokenizer_kwargs is None: + tokenizer_kwargs = {} + + # Encode the text and images + output = self.encode( + text, + images=images, + image_kwargs=image_kwargs, + tokenize=tokenize, + padding=padding, + truncation=truncation, + max_length=max_length, + add_special_tokens=False, + return_tensors=return_tensors, + **tokenizer_kwargs, + ) + return output + + @property + def model_input_names(self): + """ + Get the combined model input names from both the text tokenizer and image processor. + + Returns: + List[str]: A list of unique input names. + """ + tokenizer_input_names = self.tokenizer.model_input_names + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/modules.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/modules.py new file mode 100644 index 00000000..529e05d2 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/modules.py @@ -0,0 +1,563 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The model definition for 3D layers + +Adapted from: https://github.com/lucidrains/magvit2-pytorch/blob/9f49074179c912736e617d61b32be367eb5f993a/ +magvit2_pytorch/magvit2_pytorch.py#L889 + +[MIT License Copyright (c) 2023 Phil Wang] +https://github.com/lucidrains/magvit2-pytorch/blob/9f49074179c912736e617d61b32be367eb5f993a/LICENSE +""" + +import math +from typing import Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from cosmos1.models.autoregressive.tokenizer.patching import Patcher3D, UnPatcher3D +from cosmos1.models.autoregressive.tokenizer.utils import ( + CausalNormalize, + batch2space, + batch2time, + cast_tuple, + is_odd, + nonlinearity, + replication_pad, + space2batch, + time2batch, +) +from cosmos1.utils import log + + +class CausalConv3d(nn.Module): + def __init__( + self, + chan_in: int = 1, + chan_out: int = 1, + kernel_size: Union[int, Tuple[int, int, int]] = 3, + pad_mode: str = "constant", + **kwargs, + ): + super().__init__() + kernel_size = cast_tuple(kernel_size, 3) + + time_kernel_size, height_kernel_size, width_kernel_size = kernel_size + + assert is_odd(height_kernel_size) and is_odd(width_kernel_size) + + dilation = kwargs.pop("dilation", 1) + stride = kwargs.pop("stride", 1) + time_stride = kwargs.pop("time_stride", 1) + time_dilation = kwargs.pop("time_dilation", 1) + padding = kwargs.pop("padding", 1) + + self.pad_mode = pad_mode + time_pad = time_dilation * (time_kernel_size - 1) + (1 - time_stride) + self.time_pad = time_pad + + self.spatial_pad = (padding, padding, padding, padding) + + stride = (time_stride, stride, stride) + dilation = (time_dilation, dilation, dilation) + self.conv3d = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs) + + def _replication_pad(self, x: torch.Tensor) -> torch.Tensor: + x_prev = x[:, :, :1, ...].repeat(1, 1, self.time_pad, 1, 1) + x = torch.cat([x_prev, x], dim=2) + padding = self.spatial_pad + (0, 0) + return F.pad(x, padding, mode=self.pad_mode, value=0.0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self._replication_pad(x) + return self.conv3d(x) + + +class CausalHybridUpsample3d(nn.Module): + def __init__(self, in_channels: int, spatial_up: bool = True, temporal_up: bool = True, **ignore_kwargs) -> None: + super().__init__() + self.conv1 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(3, 1, 1), stride=1, time_stride=1, padding=0) + if temporal_up + else nn.Identity() + ) + self.conv2 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(1, 3, 3), stride=1, time_stride=1, padding=1) + if spatial_up + else nn.Identity() + ) + self.conv3 = ( + CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, time_stride=1, padding=0) + if spatial_up or temporal_up + else nn.Identity() + ) + self.spatial_up = spatial_up + self.temporal_up = temporal_up + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if not self.spatial_up and not self.temporal_up: + return x + + # hybrid upsample temporally. + if self.temporal_up: + time_factor = 1.0 + 1.0 * (x.shape[2] > 1) + if isinstance(time_factor, torch.Tensor): + time_factor = time_factor.item() + x = x.repeat_interleave(int(time_factor), dim=2) + x = x[..., int(time_factor - 1) :, :, :] + x = self.conv1(x) + x + + # hybrid upsample spatially. + if self.spatial_up: + x = x.repeat_interleave(2, dim=3).repeat_interleave(2, dim=4) + x = self.conv2(x) + x + + # final 1x1x1 conv. + x = self.conv3(x) + return x + + +class CausalHybridDownsample3d(nn.Module): + def __init__( + self, in_channels: int, spatial_down: bool = True, temporal_down: bool = True, **ignore_kwargs + ) -> None: + super().__init__() + self.conv1 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(1, 3, 3), stride=2, time_stride=1, padding=0) + if spatial_down + else nn.Identity() + ) + self.conv2 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(3, 1, 1), stride=1, time_stride=2, padding=0) + if temporal_down + else nn.Identity() + ) + self.conv3 = ( + CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, time_stride=1, padding=0) + if spatial_down or temporal_down + else nn.Identity() + ) + self.spatial_down = spatial_down + self.temporal_down = temporal_down + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if not self.spatial_down and not self.temporal_down: + return x + + # hybrid downsample spatially. + if self.spatial_down: + pad = (0, 1, 0, 1, 0, 0) + x = F.pad(x, pad, mode="constant", value=0) + x1 = self.conv1(x) + x2 = F.avg_pool3d(x, kernel_size=(1, 2, 2), stride=(1, 2, 2)) + x = x1 + x2 + + # hybrid downsample temporally. + if self.temporal_down: + x = replication_pad(x) + x1 = self.conv2(x) + x2 = F.avg_pool3d(x, kernel_size=(2, 1, 1), stride=(2, 1, 1)) + x = x1 + x2 + + # final 1x1x1 conv. + x = self.conv3(x) + return x + + +class CausalResnetBlockFactorized3d(nn.Module): + def __init__(self, *, in_channels: int, out_channels: int = None, dropout: float, num_groups: int) -> None: + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + + self.norm1 = CausalNormalize(in_channels, num_groups=1) + self.conv1 = nn.Sequential( + CausalConv3d(in_channels, out_channels, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(out_channels, out_channels, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + self.norm2 = CausalNormalize(out_channels, num_groups=num_groups) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = nn.Sequential( + CausalConv3d(out_channels, out_channels, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(out_channels, out_channels, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + self.nin_shortcut = ( + CausalConv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + if in_channels != out_channels + else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + x = self.nin_shortcut(x) + + return x + h + + +class CausalAttnBlock(nn.Module): + def __init__(self, in_channels: int, num_groups: int) -> None: + super().__init__() + + self.norm = CausalNormalize(in_channels, num_groups=num_groups) + self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + q, batch_size = time2batch(q) + k, batch_size = time2batch(k) + v, batch_size = time2batch(v) + + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) + k = k.reshape(b, c, h * w) + w_ = torch.bmm(q, k) + w_ = w_ * (int(c) ** (-0.5)) + w_ = F.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) + h_ = torch.bmm(v, w_) + h_ = h_.reshape(b, c, h, w) + + h_ = batch2time(h_, batch_size) + h_ = self.proj_out(h_) + return x + h_ + + +class CausalTemporalAttnBlock(nn.Module): + def __init__(self, in_channels: int, num_groups: int) -> None: + super().__init__() + + self.norm = CausalNormalize(in_channels, num_groups=num_groups) + self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + q, batch_size, height = space2batch(q) + k, _, _ = space2batch(k) + v, _, _ = space2batch(v) + + bhw, c, t = q.shape + q = q.permute(0, 2, 1) # (bhw, t, c) + k = k.permute(0, 2, 1) # (bhw, t, c) + v = v.permute(0, 2, 1) # (bhw, t, c) + + w_ = torch.bmm(q, k.permute(0, 2, 1)) # (bhw, t, t) + w_ = w_ * (int(c) ** (-0.5)) + + # Apply causal mask + mask = torch.tril(torch.ones_like(w_)) + w_ = w_.masked_fill(mask == 0, float("-inf")) + w_ = F.softmax(w_, dim=2) + + # attend to values + h_ = torch.bmm(w_, v) # (bhw, t, c) + h_ = h_.permute(0, 2, 1).reshape(bhw, c, t) # (bhw, c, t) + + h_ = batch2space(h_, batch_size, height) + h_ = self.proj_out(h_) + return x + h_ + + +class EncoderFactorized(nn.Module): + def __init__( + self, + in_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + spatial_compression: int, + temporal_compression: int, + **ignore_kwargs, + ) -> None: + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # Patcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.patcher3d = Patcher3D(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + in_channels = in_channels * patch_size * patch_size * patch_size + + # calculate the number of downsample operations + self.num_spatial_downs = int(math.log2(spatial_compression)) - int(math.log2(patch_size)) + assert self.num_spatial_downs <= self.num_resolutions, ( + f"Spatially downsample {self.num_resolutions} times at most" + ) + + self.num_temporal_downs = int(math.log2(temporal_compression)) - int(math.log2(patch_size)) + assert self.num_temporal_downs <= self.num_resolutions, ( + f"Temporally downsample {self.num_resolutions} times at most" + ) + + # downsampling + self.conv_in = nn.Sequential( + CausalConv3d(in_channels, channels, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(channels, channels, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + curr_res = resolution // patch_size + in_ch_mult = (1,) + tuple(channels_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = channels * in_ch_mult[i_level] + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks): + block.append( + CausalResnetBlockFactorized3d( + in_channels=block_in, out_channels=block_out, dropout=dropout, num_groups=1 + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append( + nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), CausalTemporalAttnBlock(block_in, num_groups=1) + ) + ) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + spatial_down = i_level < self.num_spatial_downs + temporal_down = i_level < self.num_temporal_downs + down.downsample = CausalHybridDownsample3d( + block_in, spatial_down=spatial_down, temporal_down=temporal_down + ) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = CausalResnetBlockFactorized3d( + in_channels=block_in, out_channels=block_in, dropout=dropout, num_groups=1 + ) + self.mid.attn_1 = nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), CausalTemporalAttnBlock(block_in, num_groups=1) + ) + self.mid.block_2 = CausalResnetBlockFactorized3d( + in_channels=block_in, out_channels=block_in, dropout=dropout, num_groups=1 + ) + + # end + self.norm_out = CausalNormalize(block_in, num_groups=1) + self.conv_out = nn.Sequential( + CausalConv3d(block_in, z_channels, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(z_channels, z_channels, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.patcher3d(x) + + # downsampling + h = self.conv_in(x) + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](h) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + if i_level != self.num_resolutions - 1: + h = self.down[i_level].downsample(h) + + # middle + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class DecoderFactorized(nn.Module): + def __init__( + self, + out_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + spatial_compression: int, + temporal_compression: int, + **ignore_kwargs, + ): + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # UnPatcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.unpatcher3d = UnPatcher3D(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + out_ch = out_channels * patch_size * patch_size * patch_size + + # calculate the number of upsample operations + self.num_spatial_ups = int(math.log2(spatial_compression)) - int(math.log2(patch_size)) + assert self.num_spatial_ups <= self.num_resolutions, f"Spatially upsample {self.num_resolutions} times at most" + self.num_temporal_ups = int(math.log2(temporal_compression)) - int(math.log2(patch_size)) + assert self.num_temporal_ups <= self.num_resolutions, ( + f"Temporally upsample {self.num_resolutions} times at most" + ) + + block_in = channels * channels_mult[self.num_resolutions - 1] + curr_res = (resolution // patch_size) // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + log.debug("Working with z of shape {} = {} dimensions.".format(self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = nn.Sequential( + CausalConv3d(z_channels, block_in, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(block_in, block_in, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + # middle + self.mid = nn.Module() + self.mid.block_1 = CausalResnetBlockFactorized3d( + in_channels=block_in, out_channels=block_in, dropout=dropout, num_groups=1 + ) + self.mid.attn_1 = nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), CausalTemporalAttnBlock(block_in, num_groups=1) + ) + self.mid.block_2 = CausalResnetBlockFactorized3d( + in_channels=block_in, out_channels=block_in, dropout=dropout, num_groups=1 + ) + + legacy_mode = ignore_kwargs.get("legacy_mode", False) + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append( + CausalResnetBlockFactorized3d( + in_channels=block_in, out_channels=block_out, dropout=dropout, num_groups=1 + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append( + nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), CausalTemporalAttnBlock(block_in, num_groups=1) + ) + ) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + # The layer index for temporal/spatial downsampling performed in the encoder should correspond + # to the layer index, inreverse order, where upsampling is performed in the decoder. + # If you've a pre-trained model, you can simply finetune. + # For example: + # Input tensor = (1, 3, 17, 32, 32) + # Patch size = 4 for 3D wavelet transform + # Compression rate = (8x16x16) + # + # We expect successive downsampling in the encoder and upsampling in the decoder to be mirrored. + # ENCODER: `(...,5,8,8) -> (...,3,4,4) -> (...,3,2,2)` + # DECODER: `(...,3,2,2) -> (...,3,4,4) -> (...,5,8,8)` + # + # if legacy_mode is True, the temporal upsampling is not perfectly mirrored. + # ENCODER: `(...,5,8,8) -> (...,3,4,4) -> (...,3,2,2)` + # DECODER: `(...,3,2,2) -> (...,5,4,4) -> (...,5,8,8)` + # + # Most of the CV and DV tokenizers were trained before 09/01/2024 with upsampling that's not mirrored. + # Going forward, new CV/DV tokenizers will adopt `legacy_mode=False`, i.e. use mirrored upsampling. + i_level_reverse = self.num_resolutions - i_level - 1 + if legacy_mode: + temporal_up = i_level_reverse < self.num_temporal_ups + else: + temporal_up = 0 < i_level_reverse < self.num_temporal_ups + 1 + spatial_up = temporal_up or ( + i_level_reverse < self.num_spatial_ups and self.num_spatial_ups > self.num_temporal_ups + ) + up.upsample = CausalHybridUpsample3d(block_in, spatial_up=spatial_up, temporal_up=temporal_up) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = CausalNormalize(block_in, num_groups=1) + self.conv_out = nn.Sequential( + CausalConv3d(block_in, out_ch, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(out_ch, out_ch, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + def forward(self, z): + h = self.conv_in(z) + + # middle block. + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # decoder blocks. + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + h = self.unpatcher3d(h) + return h diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/networks.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/networks.py new file mode 100644 index 00000000..d1f9eaad --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/networks.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from collections import namedtuple + +import torch +from torch import nn + +from cosmos1.models.autoregressive.tokenizer.modules import CausalConv3d, DecoderFactorized, EncoderFactorized +from cosmos1.models.autoregressive.tokenizer.quantizers import FSQuantizer +from cosmos1.utils import log + + +NetworkEval = namedtuple("NetworkEval", ["reconstructions", "quant_loss", "quant_info"]) + + +class CausalDiscreteVideoTokenizer(nn.Module): + def __init__(self, z_channels: int, z_factor: int, embedding_dim: int, **kwargs) -> None: + super().__init__() + self.name = kwargs.get("name", "CausalDiscreteVideoTokenizer") + self.embedding_dim = embedding_dim + self.encoder = EncoderFactorized(z_channels=z_factor * z_channels, **kwargs) + self.decoder = DecoderFactorized(z_channels=z_channels, **kwargs) + + self.quant_conv = CausalConv3d(z_factor * z_channels, embedding_dim, kernel_size=1, padding=0) + self.post_quant_conv = CausalConv3d(embedding_dim, z_channels, kernel_size=1, padding=0) + + self.quantizer = FSQuantizer(**kwargs) + + num_parameters = sum(param.numel() for param in self.parameters()) + log.debug(f"model={self.name}, num_parameters={num_parameters:,}") + log.debug(f"z_channels={z_channels}, embedding_dim={self.embedding_dim}.") + + def to(self, *args, **kwargs): + setattr(self.quantizer, "dtype", kwargs.get("dtype", torch.bfloat16)) + return super(CausalDiscreteVideoTokenizer, self).to(*args, **kwargs) + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return self.quantizer(h) + + def decode(self, quant): + quant = self.post_quant_conv(quant) + return self.decoder(quant) + + def forward(self, input): + quant_info, quant_codes, quant_loss = self.encode(input) + reconstructions = self.decode(quant_codes) + if self.training: + return dict(reconstructions=reconstructions, quant_loss=quant_loss, quant_info=quant_info) + return NetworkEval(reconstructions=reconstructions, quant_loss=quant_loss, quant_info=quant_info) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/patching.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/patching.py new file mode 100644 index 00000000..20e445f2 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/patching.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The patcher and unpatcher implementation for 2D and 3D data.""" + +import torch +import torch.nn.functional as F +from einops import rearrange + + +_WAVELETS = { + "haar": torch.tensor([0.7071067811865476, 0.7071067811865476]), + "rearrange": torch.tensor([1.0, 1.0]), +} +_PERSISTENT = False + + +class Patcher(torch.nn.Module): + """A module to convert image tensors into patches using torch operations. + + The main difference from `class Patching` is that this module implements + all operations using torch, rather than python or numpy, for efficiency purpose. + + It's bit-wise identical to the Patching module outputs, with the added + benefit of being torch.jit scriptable. + """ + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__() + self.patch_size = patch_size + self.patch_method = patch_method + self.register_buffer("wavelets", _WAVELETS[patch_method], persistent=_PERSISTENT) + self.range = range(int(torch.log2(torch.tensor(self.patch_size)).item())) + self.register_buffer("_arange", torch.arange(_WAVELETS[patch_method].shape[0]), persistent=_PERSISTENT) + for param in self.parameters(): + param.requires_grad = False + + def forward(self, x): + if self.patch_method == "haar": + return self._haar(x) + elif self.patch_method == "rearrange": + return self._arrange(x) + else: + raise ValueError("Unknown patch method: " + self.patch_method) + + def _dwt(self, x, mode="reflect", rescale=False): + dtype = x.dtype + h = self.wavelets + + n = h.shape[0] + g = x.shape[1] + hl = h.flip(0).reshape(1, 1, -1).repeat(g, 1, 1) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hh = hh.to(dtype=dtype) + hl = hl.to(dtype=dtype) + + x = F.pad(x, pad=(n - 2, n - 1, n - 2, n - 1), mode=mode).to(dtype) + xl = F.conv2d(x, hl.unsqueeze(2), groups=g, stride=(1, 2)) + xh = F.conv2d(x, hh.unsqueeze(2), groups=g, stride=(1, 2)) + xll = F.conv2d(xl, hl.unsqueeze(3), groups=g, stride=(2, 1)) + xlh = F.conv2d(xl, hh.unsqueeze(3), groups=g, stride=(2, 1)) + xhl = F.conv2d(xh, hl.unsqueeze(3), groups=g, stride=(2, 1)) + xhh = F.conv2d(xh, hh.unsqueeze(3), groups=g, stride=(2, 1)) + + out = torch.cat([xll, xlh, xhl, xhh], dim=1) + if rescale: + out = out / 2 + return out + + def _haar(self, x): + for _ in self.range: + x = self._dwt(x, rescale=True) + return x + + def _arrange(self, x): + x = rearrange(x, "b c (h p1) (w p2) -> b (c p1 p2) h w", p1=self.patch_size, p2=self.patch_size).contiguous() + return x + + +class Patcher3D(Patcher): + """A 3D discrete wavelet transform for video data, expects 5D tensor, i.e. a batch of videos.""" + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__(patch_method=patch_method, patch_size=patch_size) + self.register_buffer( + "patch_size_buffer", patch_size * torch.ones([1], dtype=torch.int32), persistent=_PERSISTENT + ) + + def _dwt(self, x, mode="reflect", rescale=False): + dtype = x.dtype + h = self.wavelets + + n = h.shape[0] + g = x.shape[1] + hl = h.flip(0).reshape(1, 1, -1).repeat(g, 1, 1) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hh = hh.to(dtype=dtype) + hl = hl.to(dtype=dtype) + + # Handles temporal axis. + x = F.pad(x, pad=(max(0, n - 2), n - 1, n - 2, n - 1, n - 2, n - 1), mode=mode).to(dtype) + xl = F.conv3d(x, hl.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + xh = F.conv3d(x, hh.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + + # Handles spatial axes. + xll = F.conv3d(xl, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xlh = F.conv3d(xl, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xhl = F.conv3d(xh, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xhh = F.conv3d(xh, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + + xlll = F.conv3d(xll, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xllh = F.conv3d(xll, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xlhl = F.conv3d(xlh, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xlhh = F.conv3d(xlh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhll = F.conv3d(xhl, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhlh = F.conv3d(xhl, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhhl = F.conv3d(xhh, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhhh = F.conv3d(xhh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + out = torch.cat([xlll, xllh, xlhl, xlhh, xhll, xhlh, xhhl, xhhh], dim=1) + if rescale: + out = out / (2 * torch.sqrt(torch.tensor(2.0))) + return out + + def _haar(self, x): + xi, xv = torch.split(x, [1, x.shape[2] - 1], dim=2) + x = torch.cat([xi.repeat_interleave(self.patch_size, dim=2), xv], dim=2) + for _ in self.range: + x = self._dwt(x, rescale=True) + return x + + def _arrange(self, x): + xi, xv = torch.split(x, [1, x.shape[2] - 1], dim=2) + x = torch.cat([xi.repeat_interleave(self.patch_size, dim=2), xv], dim=2) + x = rearrange( + x, + "b c (t p1) (h p2) (w p3) -> b (c p1 p2 p3) t h w", + p1=self.patch_size, + p2=self.patch_size, + p3=self.patch_size, + ).contiguous() + return x + + +class UnPatcher(torch.nn.Module): + """A module to convert patches into image tensorsusing torch operations. + + The main difference from `class Unpatching` is that this module implements + all operations using torch, rather than python or numpy, for efficiency purpose. + + It's bit-wise identical to the Unpatching module outputs, with the added + benefit of being torch.jit scriptable. + """ + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__() + self.patch_size = patch_size + self.patch_method = patch_method + self.register_buffer("wavelets", _WAVELETS[patch_method], persistent=_PERSISTENT) + self.range = range(int(torch.log2(torch.tensor(self.patch_size)).item())) + self.register_buffer("_arange", torch.arange(_WAVELETS[patch_method].shape[0]), persistent=_PERSISTENT) + for param in self.parameters(): + param.requires_grad = False + + def forward(self, x): + if self.patch_method == "haar": + return self._ihaar(x) + elif self.patch_method == "rearrange": + return self._iarrange(x) + else: + raise ValueError("Unknown patch method: " + self.patch_method) + + def _idwt(self, x, rescale=False): + dtype = x.dtype + h = self.wavelets + n = h.shape[0] + + g = x.shape[1] // 4 + hl = h.flip([0]).reshape(1, 1, -1).repeat([g, 1, 1]) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hh = hh.to(dtype=dtype) + hl = hl.to(dtype=dtype) + + xll, xlh, xhl, xhh = torch.chunk(x.to(dtype), 4, dim=1) + + # Inverse transform. + yl = torch.nn.functional.conv_transpose2d(xll, hl.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + yl += torch.nn.functional.conv_transpose2d(xlh, hh.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + yh = torch.nn.functional.conv_transpose2d(xhl, hl.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + yh += torch.nn.functional.conv_transpose2d(xhh, hh.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + y = torch.nn.functional.conv_transpose2d(yl, hl.unsqueeze(2), groups=g, stride=(1, 2), padding=(0, n - 2)) + y += torch.nn.functional.conv_transpose2d(yh, hh.unsqueeze(2), groups=g, stride=(1, 2), padding=(0, n - 2)) + + if rescale: + y = y * 2 + return y + + def _ihaar(self, x): + for _ in self.range: + x = self._idwt(x, rescale=True) + return x + + def _iarrange(self, x): + x = rearrange(x, "b (c p1 p2) h w -> b c (h p1) (w p2)", p1=self.patch_size, p2=self.patch_size) + return x + + +class UnPatcher3D(UnPatcher): + """A 3D inverse discrete wavelet transform for video wavelet decompositions.""" + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__(patch_method=patch_method, patch_size=patch_size) + + def _idwt(self, x, rescale=False): + dtype = x.dtype + h = self.wavelets + + g = x.shape[1] // 8 # split into 8 spatio-temporal filtered tesnors. + hl = h.flip([0]).reshape(1, 1, -1).repeat([g, 1, 1]) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hl = hl.to(dtype=dtype) + hh = hh.to(dtype=dtype) + + xlll, xllh, xlhl, xlhh, xhll, xhlh, xhhl, xhhh = torch.chunk(x, 8, dim=1) + + # Height height transposed convolutions. + xll = F.conv_transpose3d(xlll, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xll += F.conv_transpose3d(xllh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + xlh = F.conv_transpose3d(xlhl, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xlh += F.conv_transpose3d(xlhh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + xhl = F.conv_transpose3d(xhll, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhl += F.conv_transpose3d(xhlh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + xhh = F.conv_transpose3d(xhhl, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhh += F.conv_transpose3d(xhhh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + # Handles width transposed convolutions. + xl = F.conv_transpose3d(xll, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xl += F.conv_transpose3d(xlh, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xh = F.conv_transpose3d(xhl, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xh += F.conv_transpose3d(xhh, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + + # Handles time axis transposed convolutions. + x = F.conv_transpose3d(xl, hl.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + x += F.conv_transpose3d(xh, hh.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + + if rescale: + x = x * (2 * torch.sqrt(torch.tensor(2.0))) + return x + + def _ihaar(self, x): + for _ in self.range: + x = self._idwt(x, rescale=True) + x = x[:, :, self.patch_size - 1 :, ...] + return x + + def _iarrange(self, x): + x = rearrange( + x, + "b (c p1 p2 p3) t h w -> b c (t p1) (h p2) (w p3)", + p1=self.patch_size, + p2=self.patch_size, + p3=self.patch_size, + ) + x = x[:, :, self.patch_size - 1 :, ...] + return x diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/quantizers.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/quantizers.py new file mode 100644 index 00000000..589204cd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/quantizers.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Quantizers for discrete image and video tokenization.""" + +from typing import Optional + +import torch +import torch.nn as nn +from einops import rearrange + +from cosmos1.models.autoregressive.tokenizer.utils import default, pack_one, round_ste, unpack_one + + +class FSQuantizer(nn.Module): + """Finite Scalar Quantization: VQ-VAE Made Simple - https://arxiv.org/abs/2309.15505 + + Adapted from: https://github.com/lucidrains/vector-quantize-pytorch/blob/9502a1f447876d53fd37685b226bf28f250dc4a3/ + vector_quantize_pytorch/finite_scalar_quantization.py + [Copyright (c) 2020 Phil Wang] + https://github.com/lucidrains/vector-quantize-pytorch/blob/9502a1f447876d53fd37685b226bf28f250dc4a3/LICENSE + """ + + def __init__( + self, + levels: list[int], + dim: Optional[int] = None, + num_codebooks=1, + keep_num_codebooks_dim: Optional[bool] = None, + scale: Optional[float] = None, + **ignore_kwargs, + ): + super().__init__() + self.dtype = ignore_kwargs.get("dtype", torch.float32) + _levels = torch.tensor(levels, dtype=torch.int32) + self.register_buffer("_levels", _levels, persistent=False) + + _basis = torch.cumprod(torch.tensor([1] + levels[:-1]), dim=0, dtype=torch.int32) + self.register_buffer("_basis", _basis, persistent=False) + + self.scale = scale + + codebook_dim = len(levels) + self.codebook_dim = codebook_dim + + effective_codebook_dim = codebook_dim * num_codebooks + self.num_codebooks = num_codebooks + self.effective_codebook_dim = effective_codebook_dim + + keep_num_codebooks_dim = default(keep_num_codebooks_dim, num_codebooks > 1) + assert not (num_codebooks > 1 and not keep_num_codebooks_dim) + self.keep_num_codebooks_dim = keep_num_codebooks_dim + + self.dim = default(dim, len(_levels) * num_codebooks) + + has_projections = self.dim != effective_codebook_dim + self.project_in = nn.Linear(self.dim, effective_codebook_dim) if has_projections else nn.Identity() + self.project_out = nn.Linear(effective_codebook_dim, self.dim) if has_projections else nn.Identity() + self.has_projections = has_projections + + self.codebook_size = self._levels.prod().item() + + implicit_codebook = self.indices_to_codes(torch.arange(self.codebook_size), project_out=False) + self.register_buffer("implicit_codebook", implicit_codebook, persistent=False) + + def bound(self, z: torch.Tensor, eps: float = 1e-3) -> torch.Tensor: + """Bound `z`, an array of shape (..., d).""" + half_l = (self._levels - 1) * (1 + eps) / 2 + offset = torch.where(self._levels % 2 == 0, 0.5, 0.0) + shift = (offset / half_l).atanh() + return (z + shift).tanh() * half_l - offset + + def quantize(self, z: torch.Tensor) -> torch.Tensor: + """Quantizes z, returns quantized zhat, same shape as z.""" + quantized = round_ste(self.bound(z)) + half_width = self._levels // 2 # Renormalize to [-1, 1]. + return quantized / half_width + + def _scale_and_shift(self, zhat_normalized: torch.Tensor) -> torch.Tensor: + half_width = self._levels // 2 + return (zhat_normalized * half_width) + half_width + + def _scale_and_shift_inverse(self, zhat: torch.Tensor) -> torch.Tensor: + half_width = self._levels // 2 + return (zhat - half_width) / half_width + + def codes_to_indices(self, zhat: torch.Tensor) -> torch.Tensor: + """Converts a `code` to an index in the codebook.""" + assert zhat.shape[-1] == self.codebook_dim + zhat = self._scale_and_shift(zhat).float() + return (zhat * self._basis).sum(dim=-1).to(torch.int32) + + def indices_to_codes(self, indices: torch.Tensor, project_out=True) -> torch.Tensor: + """Inverse of `codes_to_indices`.""" + is_img_or_video = indices.ndim >= (3 + int(self.keep_num_codebooks_dim)) + indices = rearrange(indices, "... -> ... 1") + codes_non_centered = (indices // self._basis) % self._levels + codes = self._scale_and_shift_inverse(codes_non_centered) + + if self.keep_num_codebooks_dim: + codes = rearrange(codes, "... c d -> ... (c d)") + + if project_out: + codes = self.project_out(codes) + + if is_img_or_video: + codes = rearrange(codes, "b ... d -> b d ...") + + return codes.to(self.dtype) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + """ + einstein notation + b - batch + n - sequence (or flattened spatial dimensions) + d - feature dimension, which is also log2(codebook size) + c - number of codebook dim + """ + is_img_or_video = z.ndim >= 4 + + # standardize image or video into (batch, seq, dimension) + + if is_img_or_video: + z = rearrange(z, "b d ... -> b ... d") + z, ps = pack_one(z, "b * d") + + assert z.shape[-1] == self.dim, f"expected dimension of {self.dim} but found dimension of {z.shape[-1]}" + + z = self.project_in(z) + + z = rearrange(z, "b n (c d) -> b n c d", c=self.num_codebooks) + + codes = self.quantize(z) + indices = self.codes_to_indices(codes) + + codes = rearrange(codes, "b n c d -> b n (c d)") + + out = self.project_out(codes) + + # reconstitute image or video dimensions + + if is_img_or_video: + out = unpack_one(out, ps, "b * d") + out = rearrange(out, "b ... d -> b d ...") + indices = unpack_one(indices, ps, "b * c") + dummy_loss = torch.zeros_like(out.mean(dim=[1, 2, 3], keepdim=True)) + else: + dummy_loss = torch.zeros_like(out.mean(dim=[1, 2], keepdim=True)).unsqueeze(1) + + if not self.keep_num_codebooks_dim: + indices = rearrange(indices, "... 1 -> ...") + + return (indices, out.to(self.dtype), dummy_loss) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/text_tokenizer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/text_tokenizer.py new file mode 100644 index 00000000..43fad675 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/text_tokenizer.py @@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import AutoTokenizer + +from cosmos1.utils import log + + +def get_tokenizer_path(model_family: str, is_instruct_model: bool = False): + """ + Get the tokenizer path from the model family and instruct model flag. + Args: + model_family (str): The model family. + is_instruct_model (bool): Whether the model is an instruct model. + Returns: + str: The tokenizer path in s3. + """ + model_family = model_family.lower() + if model_family == "mistral": + return "mistralai/Mistral-Nemo-Instruct-2407" + else: + assert model_family in ["llama3", "llama3.1"] + if model_family == "llama3": + model_path = "meta-llama/Meta-Llama-3-8B" + elif model_family == "llama3.1": + model_path = "meta-llama/Llama-3.1-8B" + else: + raise ValueError(f"Unsupported model family: {model_family}") + suffix = "-Instruct" if is_instruct_model else "" + model_path = f"{model_path}{suffix}" + return model_path + + +class TextTokenizer: + """ + Text tokenizer class built on HuggingFace's Fast Tokenizer (Rust based). + """ + + def __init__( + self, + model_family: str, + is_instruct_model: bool, + local_path: Optional[str] = None, + ): + """ + Initialize the TextTokenizer. + Args: + model_family (str): The model family. + is_instruct_model (bool): Whether the model is an instruct model. + local_path (Optional[str]): The local path to the tokenizer. If not provided, the tokenizer will be downloaded from the remote path. + """ + if local_path is None: + tokenizer_path = get_tokenizer_path(model_family, is_instruct_model) + else: + tokenizer_path = local_path + + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True) + self.stop_tokens = { + self.tokenizer.eos_token_id, + } + self.model_family = model_family + self.is_instruct_model = is_instruct_model + self.eos_id = self.tokenizer.eos_token_id + if self.tokenizer.pad_token is None: + if model_family.startswith("llama"): + self.pad_id = 128004 # "<|finetune_right_pad_id|>" + elif model_family == "mistral": + self.pad_id = 10 # "" + elif model_family == "pixtral": + self.pad_id = 11 # "" + else: + raise ValueError(f"pad_id not defined for model_family {model_family}") + else: + self.pad_id = self.tokenizer.pad_token_id + + def tokenize(self, text: str, *, add_special_tokens: bool = False, **kwargs) -> List[str]: + """ + Converts a string into a sequence of tokens, replacing unknown tokens with the `unk_token`. + + Args: + text (`str`): + The sequence to be encoded. + add_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not to add the special tokens associated with the corresponding model. + Returns: + `List[str]`: The list of tokens. + """ + return self.tokenizer.tokenize(text, add_special_tokens=add_special_tokens, **kwargs) + + def encode( + self, + text: Union[str, List[str], List[int]], + *, # Enforce keyword-only arguments + add_special_tokens: bool = True, + padding: Union[bool, str] = False, + truncation: Union[bool, str] = None, + max_length: Optional[int] = None, + stride: int = 0, + return_tensors: Optional[str] = None, + **kwargs, + ) -> List[int]: + """ + Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary. + + Args: + text (`str`, `List[str]` or `List[int]`): + The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the + `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method). + add_special_tokens (`bool`, *optional*, defaults to `True`): + Whether or not to add special tokens when encoding the sequences. This will use the underlying + `PretrainedTokenizerBase.build_inputs_with_special_tokens` function, which defines which tokens are + automatically added to the input ids. This is usefull if you want to add `bos` or `eos` tokens + automatically. + padding (`bool`, `str`, *optional*, defaults to `False`): + Activates and controls padding. Accepts the following values: + + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + truncation (`bool`, `str`, *optional*, defaults to `False`): + Activates and controls truncation. Accepts the following values: + + - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or + to the maximum acceptable input length for the model if that argument is not provided. This will + truncate token by token, removing a token from the longest sequence in the pair if a pair of + sequences (or a batch of pairs) is provided. + - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the + maximum acceptable input length for the model if that argument is not provided. This will only + truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided. + - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the + maximum acceptable input length for the model if that argument is not provided. This will only + truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided. + - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths + greater than the model maximum admissible input size). + max_length (`int`, *optional*): + Controls the maximum length to use by one of the truncation/padding parameters. + + If left unset or set to `None`, this will use the predefined model maximum length if a maximum length + is required by one of the truncation/padding parameters. If the model has no specific maximum input + length (like XLNet) truncation/padding to a maximum length will be deactivated. + stride (`int`, *optional*, defaults to 0): + If set to a number along with `max_length`, the overflowing tokens returned when + `return_overflowing_tokens=True` will contain some tokens from the end of the truncated sequence + returned to provide some overlap between truncated and overflowing sequences. The value of this + argument defines the number of overlapping tokens. + is_split_into_words (`bool`, *optional*, defaults to `False`): + Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the + tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) + which it will tokenize. This is useful for NER or token classification. + pad_to_multiple_of (`int`, *optional*): + If set will pad the sequence to a multiple of the provided value. Requires `padding` to be activated. + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability + `>= 7.5` (Volta). + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors instead of list of python integers. Acceptable values are: + + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return Numpy `np.ndarray` objects. + """ + return self.tokenizer.encode( + text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + stride=stride, + return_tensors=return_tensors, + ) + + def decode( + self, + token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor"], + *, # Enforce keyword-only arguments + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool = None, + **kwargs, + ) -> str: + """ + Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special + tokens and clean up tokenization spaces. + + Args: + token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): + List of tokenized input ids. Can be obtained using the `__call__` method. + skip_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not to remove special tokens in the decoding. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces`. + kwargs (additional keyword arguments, *optional*): + Will be passed to the underlying model specific decode method. + + Returns: + `str`: The decoded sentence. + """ + return self.tokenizer.decode( + token_ids, + skip_special_tokens=skip_special_tokens, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + + def apply_chat_template( + self, + conversation: Union[List[Dict[str, str]], List[List[Dict[str, str]]]], + *, + add_generation_prompt: bool = False, + tokenize: bool = True, + padding: bool = False, + truncation: bool = False, + max_length: Optional[int] = None, + return_tensors: Optional[str] = None, + return_dict: bool = False, + return_assistant_tokens_mask: bool = False, + generation_prefix: str = "", + tokenizer_kwargs: Optional[Dict[str, Any]] = None, + **kwargs, + ): + """ + Converts a list of dictionaries with `"role"` and `"content"` keys to a list of token + ids. This method is intended for use with chat models, and will read the tokenizer's chat_template attribute to determine the format and control tokens to use when converting. + + More details can be found at https://huggingface.co/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.apply_chat_template + + Args: + conversation (Union[List[Dict[str, str]], List[List[Dict[str, str]]]]): A list of dicts + with "role" and "content" keys, representing the chat history so far. + add_generation_prompt (bool, *optional*): + If this is set, a prompt with the token(s) that indicate + the start of an assistant message will be appended to the formatted output. This is useful when you want to generate a response from the model. + Note that this argument will be passed to the chat template, and so it must be supported in the + template for this argument to have any effect. + continue_final_message (bool, *optional*): + If this is set, the chat will be formatted so that the final + message in the chat is open-ended, without any EOS tokens. The model will continue this message + rather than starting a new one. This allows you to "prefill" part of + the model's response for it. Cannot be used at the same time as `add_generation_prompt`. + tokenize (`bool`, defaults to `True`): + Whether to tokenize the output. If `False`, the output will be a string. + padding (`bool`, defaults to `False`): + Whether to pad sequences to the maximum length. Has no effect if tokenize is `False`. + truncation (`bool`, defaults to `False`): + Whether to truncate sequences at the maximum length. Has no effect if tokenize is `False`. + max_length (`int`, *optional*): + Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is `False`. If + not specified, the tokenizer's `max_length` attribute will be used as a default. + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors of a particular framework. Has no effect if tokenize is `False`. Acceptable + values are: + - `'tf'`: Return TensorFlow `tf.Tensor` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return NumPy `np.ndarray` objects. + - `'jax'`: Return JAX `jnp.ndarray` objects. + return_dict (`bool`, defaults to `False`): + Whether to return a dictionary with named outputs. Has no effect if tokenize is `False`. + generation_prefix (str): Prefix to add before asking model to generate. Helpful to guide the generation. Defaults to "". + tokenizer_kwargs (`Dict[str: Any]`, *optional*): Additional kwargs to pass to the tokenizer. + return_assistant_tokens_mask (`bool`, defaults to `False`): + Whether to return a mask of the assistant generated tokens. For tokens generated by the assistant, + the mask will contain 1. For user and system tokens, the mask will contain 0. + This functionality is only available for chat templates that support it via the `{% generation %}` keyword. + **kwargs: Additional kwargs to pass to the template renderer. Will be accessible by the chat template. + + Returns: + `Union[List[int], Dict]`: A list of token ids representing the tokenized chat so far, including control tokens. This + output is ready to pass to the model, either directly or via methods like `generate()`. If `return_dict` is + set, will return a dict of tokenizer outputs instead. + """ + if not self.is_instruct_model: + raise ValueError( + "apply_chat_template is only supported for instruct models. You should pass argument is_instruct_model=True to the TextTokenizer constructor." + ) + # Since generation_prefix is added to the text in the end, ensure that the setting is correct + if generation_prefix: + assert not tokenize, "tokenize must be False when generation_prefix is provided." + assert add_generation_prompt, "add_generation_prompt must be set when generation_prefix is provided." + formatted_text: Union[str, List[int]] = self.tokenizer.apply_chat_template( + conversation, + add_generation_prompt=add_generation_prompt, + tokenize=tokenize, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_dict=return_dict, + return_assistant_tokens_mask=return_assistant_tokens_mask, + tokenizer_kwargs=tokenizer_kwargs, + **kwargs, + ) + if generation_prefix: + formatted_text: str = formatted_text + generation_prefix + log.debug( + f"Adding generation prefix: {generation_prefix} to the formatted text\n" + f"Formatted text: {formatted_text}" + ) + return formatted_text diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/tokenizer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/tokenizer.py new file mode 100644 index 00000000..769206cf --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/tokenizer.py @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from collections import defaultdict +from typing import Optional + +import torch +from einops import rearrange + +from cosmos1.models.autoregressive.configs.base.tokenizer import TokenizerConfig +from cosmos1.utils.lazy_config import instantiate as lazy_instantiate + + +def update_vocab_size( + existing_vocab_size, + to_be_added_vocab_size, + training_type, + add_special_tokens, + video_special_tokens={}, +): + # New vocab size + if add_special_tokens: + existing_vocab_size += to_be_added_vocab_size + len(video_special_tokens) + # For text_to_video, we add one special token at the beginning of the video + elif training_type == "text_to_video": + existing_vocab_size += to_be_added_vocab_size + 1 + else: + existing_vocab_size += to_be_added_vocab_size + return existing_vocab_size + + +class DiscreteMultimodalTokenizer: + def __init__(self, tokenizer_config: TokenizerConfig): + self.tokenizer_config = tokenizer_config + self.vocab_size = 0 + self.total_seq_len = tokenizer_config.seq_len + self.pad_to_multiple_of = tokenizer_config.pad_to_multiple_of + self.training_type = tokenizer_config.training_type + assert self.training_type in [ + "text_only", + "text_to_video", + "video_to_video", + "image_text_interleaved", + ], f"{self.training_type} not supported" + + self._build_text_tokenizer() + self._build_video_tokenizer() + + def _build_text_tokenizer(self): + r"""Function to initialize the text tokenizer model.""" + if self.tokenizer_config.text_tokenizer is not None: + self.text_tokenizer = lazy_instantiate(self.tokenizer_config.text_tokenizer.config) + self.vocab_size += self.tokenizer_config.text_tokenizer.vocab_size + else: + self.text_tokenizer = None + + def _build_video_tokenizer(self): + r"""Function to initialize the video tokenizer model.""" + if self.tokenizer_config.video_tokenizer is not None: + self.video_tokenizer = lazy_instantiate(self.tokenizer_config.video_tokenizer.config) + self.video_tokenizer = self.video_tokenizer.to("cuda") + self.video_vocab_size = self.tokenizer_config.video_tokenizer.vocab_size + special_token_offset = ( + self.tokenizer_config.video_tokenizer.tokenizer_offset + + self.tokenizer_config.video_tokenizer.vocab_size + ) + self.video_special_tokens = { + "<|begin_of_video|>": special_token_offset, + "<|end_of_video|>": special_token_offset + 1, + "<|pad_token_video|>": special_token_offset + 2, + } + + self.vocab_size = update_vocab_size( + existing_vocab_size=self.vocab_size, + to_be_added_vocab_size=self.tokenizer_config.video_tokenizer.vocab_size, + training_type=self.training_type, + add_special_tokens=self.tokenizer_config.add_special_tokens, + video_special_tokens=self.video_special_tokens, + ) + else: + self.video_tokenizer = None + + @property + def pad_id(self): + r"""Returns the pad_id.""" + + if self.training_type == "text_only" or self.training_type == "image_text_interleaved": + pad_id = self.text_tokenizer.pad_id + elif self.training_type in ["text_to_video", "video_to_video"]: + pad_id = self.video_special_tokens["<|pad_token_video|>"] + else: + raise ValueError(f"training_type {self.training_type} not defined") + return pad_id + + @property + def ignore_index(self): + r"""Returns which token should be ignored during loss computation.""" + if self.training_type == "text_only" or self.training_type == "image_text_interleaved": + if self.text_tokenizer.pad_id == self.text_tokenizer.eos_id: + # If the PAD token is the same as the EOS token, we do not ignore it during loss + # computation, since we want the model to be able to predict EOS tokens in inference. + # The PyTorch default ignore_index for the cross-entropy loss is -100. + ignore_index = -100 + else: + ignore_index = self.text_tokenizer.pad_id + elif self.training_type in ["text_to_video", "video_to_video"]: + ignore_index = self.pad_id + else: + raise ValueError(f"training_type {self.training_type} not defined") + return ignore_index + + @property + def stop_tokens(self): + r"""Returns the stop tokens.""" + if self.training_type == "text_only" or self.training_type == "image_text_interleaved": + stop_tokens = self.text_tokenizer.stop_tokens + elif self.training_type in ["text_to_video", "video_to_video"]: + stop_tokens = set([self.video_special_tokens["<|end_of_video|>"]]) + else: + raise ValueError(f"training_type {self.training_type} not defined") + return stop_tokens + + def _tokenize_text(self, raw_text: list[str], max_text_seq_len: int = -1): + r"""Function to tokenize text. + Args: + raw_text (list[str]): List of input strings + max_text_seq_len (int): Maximum sequence length returned by text tokenizer + Returns: + text_tokens (list[list[int]]): List of text tokens + """ + + batch_size = len(raw_text) + text_tokens = [self.text_tokenizer.encode(raw_text[i], bos=True, eos=True) for i in range(batch_size)] + + # Clipping the text tokens so that the sequence length does not exceed max_text_seq_len + if max_text_seq_len > -1: + for i in range(len(text_tokens)): + if len(text_tokens[i]) > max_text_seq_len: + # Simply clip and add end of seq token + text_tokens[i] = text_tokens[i][0 : max_text_seq_len - 1] + [self.text_tokenizer.eos_id] + return text_tokens + + def _tokenize_class(self, cls_labels: list[str]): + r"""Function to tokenize the class label. + Args: + cls_labels (list[str]): List of class indices + Returns: + class_tokens (list[list[int]]): List of class tokens + """ + + # tokenizer_offset tells what offset should be added to the tokens. + # This is needed for vocab expansion. + class_tokens = [[int(x) + self.tokenizer_config.class_tokenizer.tokenizer_offset] for x in cls_labels] + + return class_tokens + + def _tokenize_video(self, videos: torch.Tensor, pixel_chunk_duration: Optional[int] = None): + r"""Function to tokenize video. + Args: + videos (torch.Tensor): Input video data tensor + pixel_chunk_duration (Optional[float]): Pixel chunk duration. If provided, we pass it to the video tokenizer. + Returns: + video_tokens (list[list[int]]): List of video tokens + """ + + video_tokens = [] + batch_size = videos.shape[0] + + quantized_out, _ = self.video_tokenizer.encode(videos, pixel_chunk_duration=pixel_chunk_duration) + indices = self.video_tokenizer.fsq_quantizer.codes_to_indices(quantized_out.permute(0, 2, 3, 4, 1)) + + # Flatten the indices + indices = rearrange(indices, "B T H W -> B (T H W)") + + # tokenizer_offset tells what offset should be added to the tokens. + # This is needed for vocab expansion. + indices += self.tokenizer_config.video_tokenizer.tokenizer_offset + + # Add begin and end of video tokens + bov_token = self.video_special_tokens["<|begin_of_video|>"] + eov_token = self.video_special_tokens["<|end_of_video|>"] + + # Append bov and eov tokens + if self.tokenizer_config.add_special_tokens: + for i in range(batch_size): + video_tokens.append([bov_token] + indices[i].tolist() + [eov_token]) + else: + if self.training_type == "text_to_video": + for i in range(batch_size): + video_tokens.append([bov_token] + indices[i].tolist()) + else: + for i in range(batch_size): + video_tokens.append(indices[i].tolist()) + assert len(video_tokens[-1]) == self.tokenizer_config.video_tokenizer.max_seq_len, ( + f"Expected {self.tokenizer_config.video_tokenizer.max_seq_len} tokens, got {len(video_tokens[-1])}; video shape: {videos.shape}" + ) + + return video_tokens + + def tokenize(self, data_batch: dict): + r"""Function to tokenize data_dict. + Args: + data_batch (dict): Input data dict + Returns: + tokens (torch.LongTensor): Token tensor dict + """ + + if ( + self.training_type in ["text_only", "image_text_interleaved"] + and not self.tokenizer_config.text_tokenizer.tokenize_here + ): + # In case of pre-computed tokens, just return the data_batch + return data_batch["tokens"], None + + # Online tokenization + tokens = [] + token_boundaries = defaultdict(list) + + # Obtain maximum sequence length + max_text_seq_len = -1 + max_visual_seq_len = -1 + + if self.training_type in ["text_to_video", "video_to_video"]: + max_visual_seq_len = self.tokenizer_config.video_tokenizer.max_seq_len + + # If max visual sequence length is specified, make sure that text is clipped so that + # the full video/image is always seen. + if max_visual_seq_len > -1: + if self.tokenizer_config.add_special_tokens: + max_visual_seq_len = max_visual_seq_len + 2 # Two special tokens is for [bov, eov] or [boi, eoi] token + elif self.training_type == "text_to_video": + max_visual_seq_len = max_visual_seq_len + 1 + else: + pass + assert max_visual_seq_len <= self.total_seq_len, ( + f"max_visual_seq_len ({max_visual_seq_len}) is greater that total sequence length ({self.total_seq_len})" + ) + max_text_seq_len = self.total_seq_len - max_visual_seq_len + + # Tokenize the text + if ( + "text" in self.training_type + and self.text_tokenizer is not None + and self.tokenizer_config.text_tokenizer.tokenize_here + ): + key = self.tokenizer_config.text_tokenizer.data_key + batch_size = len(data_batch[key]) + assert key in data_batch, f"Key {key} should be present in data for text tokenizer" + tokens = self._tokenize_text(data_batch["caption"], max_text_seq_len) + + for i in range(batch_size): + token_boundaries["text"].append((0, len(tokens[i]))) + else: + tokens = [] + batch_size = None + + # Tokenize the class label + if "class" in self.training_type and self.tokenizer_config.class_tokenizer is not None: + key = self.tokenizer_config.class_tokenizer.data_key + assert key in data_batch, f"Key {key} should be present in data for class tokenizer" + batch_size = len(data_batch[key]) if batch_size is None else batch_size + tokens_class = self._tokenize_class(data_batch[key]) + if len(tokens) == 0: + tokens = tokens_class + for i in range(batch_size): + token_boundaries["class"].append((0, len(tokens[i]))) + else: + for i in range(batch_size): + token_boundaries["class"].append((len(tokens[i]), len(tokens[i]) + len(tokens_class[i]))) + tokens[i] = tokens[i] + tokens_class[i] + + # Tokenize the video + if self.video_tokenizer is not None and self.tokenizer_config.video_tokenizer.tokenize_here: + key = self.tokenizer_config.video_tokenizer.data_key + assert key in data_batch, f"Key {key} should be present in data for video tokenizer" + batch_size = len(data_batch[key]) if batch_size is None else batch_size + + pixel_chunk_duration = ( + None # If not specified, we assume it's a video dataset and use the default chunk duration + ) + dataset_name = data_batch.get("dataset_name", None) + if dataset_name is not None and dataset_name.startswith("image"): + # If it's an image dataset, we use a pixel chunk duration of 1 + pixel_chunk_duration = 1 + tokens_video = self._tokenize_video(data_batch[key], pixel_chunk_duration=pixel_chunk_duration) + if len(tokens) == 0: + tokens = tokens_video + for i in range(batch_size): + token_boundaries["video"].append((0, len(tokens[i]))) + # [B,] each entry is ((0, len(tokens[i]))) + else: + for i in range(batch_size): + token_boundaries["video"].append((len(tokens[i]), len(tokens[i]) + len(tokens_video[i]))) + tokens[i] = tokens[i] + tokens_video[i] + + # Combine the tokens and do padding + max_seq_len_in_batch = max([len(token) for token in tokens]) + if self.pad_to_multiple_of is not None: + # Pad the sequence length to the nearest multiple of pad_to_multiple_of + max_seq_len_in_batch = ( + (max_seq_len_in_batch - 1) // self.pad_to_multiple_of + 1 + ) * self.pad_to_multiple_of + pad_to_len = min(max_seq_len_in_batch, self.total_seq_len) + for i in range(len(tokens)): + if len(tokens[i]) < pad_to_len: + tokens[i] = tokens[i] + [self.pad_id] * (pad_to_len - len(tokens[i])) + else: + tokens[i] = tokens[i][0:pad_to_len] + + # Convert it to long tensor + tokens = torch.LongTensor(tokens) + return tokens, token_boundaries diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/utils.py new file mode 100644 index 00000000..093cd207 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/tokenizer/utils.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any + +import torch +from einops import pack, rearrange, unpack + + +def time2batch(x: torch.Tensor) -> tuple[torch.Tensor, int]: + batch_size = x.shape[0] + return rearrange(x, "b c t h w -> (b t) c h w"), batch_size + + +def batch2time(x: torch.Tensor, batch_size: int) -> torch.Tensor: + return rearrange(x, "(b t) c h w -> b c t h w", b=batch_size) + + +def space2batch(x: torch.Tensor) -> tuple[torch.Tensor, int]: + batch_size, height = x.shape[0], x.shape[-2] + return rearrange(x, "b c t h w -> (b h w) c t"), batch_size, height + + +def batch2space(x: torch.Tensor, batch_size: int, height: int) -> torch.Tensor: + return rearrange(x, "(b h w) c t -> b c t h w", b=batch_size, h=height) + + +def cast_tuple(t: Any, length: int = 1) -> Any: + return t if isinstance(t, tuple) else ((t,) * length) + + +def replication_pad(x): + return torch.cat([x[:, :, :1, ...], x], dim=2) + + +def divisible_by(num: int, den: int) -> bool: + return (num % den) == 0 + + +def is_odd(n: int) -> bool: + return not divisible_by(n, 2) + + +def nonlinearity(x): + return x * torch.sigmoid(x) + + +class CausalNormalize(torch.nn.Module): + def __init__(self, in_channels, num_groups=1): + super().__init__() + self.norm = torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + self.num_groups = num_groups + + def forward(self, x): + # if num_groups !=1, we apply a spatio-temporal groupnorm for backward compatibility purpose. + # All new models should use num_groups=1, otherwise causality is not guaranteed. + if self.num_groups == 1: + x, batch_size = time2batch(x) + return batch2time(self.norm(x), batch_size) + return self.norm(x) + + +def exists(v): + return v is not None + + +def default(*args): + for arg in args: + if exists(arg): + return arg + return None + + +def pack_one(t, pattern): + return pack([t], pattern) + + +def unpack_one(t, ps, pattern): + return unpack(t, ps, pattern)[0] + + +def round_ste(z: torch.Tensor) -> torch.Tensor: + """Round with straight through gradients.""" + zhat = z.round() + return z + (zhat - z).detach() + + +def log(t, eps=1e-5): + return t.clamp(min=eps).log() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/checkpoint.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/checkpoint.py new file mode 100644 index 00000000..aa0b0e18 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/checkpoint.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +import torch + + +# Substrings to ignore when processing state dicts +substrings_to_ignore = [ + "_extra_state", # Extra states (BytesIO type) added by TransformerEngine for FP8 handling +] + + +def get_partial_state_dict( + state_dict: Dict[str, torch.Tensor], + prefix: str, +) -> Dict[str, torch.Tensor]: + """ + Get a partial state dict with keys starting with the given prefix + """ + return {k: v for k, v in state_dict.items() if k.startswith(prefix)} + + +def process_state_dict( + state_dict: Dict[str, torch.Tensor], + device: str = None, + dtype: torch.dtype = None, + prefix_to_remove: Optional[str] = None, +) -> Dict[str, torch.Tensor]: + """ + - Remove items with substring "_extra_state" in keys (TransformerEngine adds these for FP8) + - Move tensors to specified device and dtype if provided + + Args: + state_dict (Dict[str, torch.Tensor]): The state dict to process + device (str, optional): The device to move tensors to. Defaults to None. + dtype (torch.dtype, optional): The dtype to move tensors to. Defaults to None. + prefix_to_remove (str, optional): The prefix to remove from the keys of the state dict. Defaults to None. + + Returns: + Dict[str, torch.Tensor]: The processed state dict + """ + new_state_dict = {} + tensor_kwargs = {} + if device is not None: + tensor_kwargs["device"] = device + if dtype is not None: + tensor_kwargs["dtype"] = dtype + + for key, value in state_dict.items(): + # Check if any of the substrings to ignore are in the key + skip = False + for substr in substrings_to_ignore: + if substr in key: + skip = True + break + if skip: + continue + if len(tensor_kwargs) > 0: + value = value.to(**tensor_kwargs) + if prefix_to_remove is not None and key.startswith(prefix_to_remove): + key = key[len(prefix_to_remove) :] + new_state_dict[key] = value + return new_state_dict diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/inference.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/inference.py new file mode 100644 index 00000000..329c662c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/inference.py @@ -0,0 +1,393 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import json +import math +import os +from pathlib import Path +from typing import List + +import numpy as np +import torch +import torchvision +from PIL import Image + +from cosmos1.models.autoregressive.configs.inference.inference_config import SamplingConfig +from cosmos1.utils import log + + +_IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", "webp"] +_VIDEO_EXTENSIONS = [".mp4"] +_SUPPORTED_CONTEXT_LEN = [1, 9] # Input frames +NUM_TOTAL_FRAMES_DEFAULT = 33 + + +def add_common_arguments(parser): + """Add common command line arguments. + + Args: + parser (ArgumentParser): Argument parser to add arguments to + """ + parser.add_argument( + "--checkpoint_dir", type=str, default="checkpoints", help="Base directory containing model checkpoints" + ) + parser.add_argument( + "--video_save_name", + type=str, + default="output", + help="Output filename for generating a single video", + ) + parser.add_argument("--video_save_folder", type=str, default="outputs/", help="Output folder for saving videos") + parser.add_argument( + "--input_image_or_video_path", + type=str, + help="Input path for input image or video", + ) + parser.add_argument( + "--batch_input_path", + type=str, + help="Input folder containing all input images or videos", + ) + parser.add_argument( + "--num_input_frames", + type=int, + default=9, + help="Number of input frames for world generation", + choices=_SUPPORTED_CONTEXT_LEN, + ) + parser.add_argument("--temperature", type=float, default=1.0, help="Temperature for sampling") + parser.add_argument("--top_p", type=float, default=0.8, help="Top-p value for sampling") + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument("--disable_diffusion_decoder", action="store_true", help="Disable diffusion decoder") + parser.add_argument( + "--offload_guardrail_models", + action="store_true", + help="Offload guardrail models after inference", + ) + parser.add_argument( + "--offload_diffusion_decoder", + action="store_true", + help="Offload diffusion decoder after inference", + ) + parser.add_argument( + "--offload_ar_model", + action="store_true", + help="Offload AR model after inference", + ) + parser.add_argument( + "--offload_tokenizer", + action="store_true", + help="Offload discrete tokenizer model after inference", + ) + + +def validate_args(args: argparse.Namespace, inference_type: str): + """Validate command line arguments for base and video2world generation.""" + assert inference_type in [ + "base", + "video2world", + ], "Invalid inference_type, must be 'base' or 'video2world'" + if args.input_type in ["image", "text_and_image"] and args.num_input_frames != 1: + args.num_input_frames = 1 + log.info(f"Set num_input_frames to 1 for {args.input_type} input") + + if args.num_input_frames == 1: + if "4B" in args.ar_model_dir: + log.warning( + "The failure rate for 4B model with image input is ~15%. 12B / 13B model have a smaller failure rate. Please be cautious and refer to README.md for more details." + ) + elif "5B" in args.ar_model_dir: + log.warning( + "The failure rate for 5B model with image input is ~7%. 12B / 13B model have a smaller failure rate. Please be cautious and refer to README.md for more details." + ) + + # Validate prompt/image/video args for single or batch generation + assert args.input_image_or_video_path or args.batch_input_path, ( + "--input_image_or_video_path or --batch_input_path must be provided." + ) + if inference_type == "video2world" and (not args.batch_input_path): + assert args.prompt, "--prompt is required for single video generation." + args.data_resolution = [640, 1024] + + # Validate number of GPUs + num_gpus = int(os.getenv("WORLD_SIZE", 1)) + assert num_gpus <= 1, "We support only single GPU inference for now" + + # Create output folder + Path(args.video_save_folder).mkdir(parents=True, exist_ok=True) + + sampling_config = SamplingConfig( + echo=True, + temperature=args.temperature, + top_p=args.top_p, + compile_sampling=True, + ) + return sampling_config + + +def resize_input(video: torch.Tensor, resolution: list[int]): + r""" + Function to perform aspect ratio preserving resizing and center cropping. + This is needed to make the video into target resolution. + Args: + video (torch.Tensor): Input video tensor + resolution (list[int]): Data resolution + Returns: + Cropped video + """ + + orig_h, orig_w = video.shape[2], video.shape[3] + target_h, target_w = resolution + + scaling_ratio = max((target_w / orig_w), (target_h / orig_h)) + resizing_shape = (int(math.ceil(scaling_ratio * orig_h)), int(math.ceil(scaling_ratio * orig_w))) + video_resized = torchvision.transforms.functional.resize(video, resizing_shape) + video_cropped = torchvision.transforms.functional.center_crop(video_resized, resolution) + return video_cropped + + +def load_image_from_list(flist, data_resolution: List[int], num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT) -> dict: + """ + Function to load images from a list of image paths. + Args: + flist (List[str]): List of image paths + data_resolution (List[int]): Data resolution + Returns: + Dict containing input images + """ + all_videos = dict() + for img_path in flist: + ext = os.path.splitext(img_path)[1] + if ext in _IMAGE_EXTENSIONS: + # Read the image + img = Image.open(img_path) + + # Convert to tensor + img = torchvision.transforms.functional.to_tensor(img) + static_vid = img.unsqueeze(0).repeat(num_total_frames, 1, 1, 1) + static_vid = static_vid * 2 - 1 + + log.debug( + f"Resizing input image of shape ({static_vid.shape[2]}, {static_vid.shape[3]}) -> ({data_resolution[0]}, {data_resolution[1]})" + ) + static_vid = resize_input(static_vid, data_resolution) + fname = os.path.basename(img_path) + all_videos[fname] = static_vid.transpose(0, 1).unsqueeze(0) + + return all_videos + + +def read_input_images( + batch_input_path: str, data_resolution: List[int], num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT +) -> dict: + """ + Function to read input images from a JSONL file. + + Args: + batch_input_path (str): Path to JSONL file containing visual input paths + data_resolution (list[int]): Data resolution + + Returns: + Dict containing input images + """ + # Read visual inputs from JSONL + flist = [] + with open(batch_input_path, "r") as f: + for line in f: + data = json.loads(line.strip()) + flist.append(data["visual_input"]) + + return load_image_from_list(flist, data_resolution=data_resolution, num_total_frames=num_total_frames) + + +def read_input_image( + input_path: str, data_resolution: List[int], num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT +) -> dict: + """ + Function to read input image. + Args: + input_path (str): Path to input image + data_resolution (List[int]): Data resolution + Returns: + Dict containing input image + """ + flist = [input_path] + return load_image_from_list(flist, data_resolution=data_resolution, num_total_frames=num_total_frames) + + +def read_input_videos( + batch_input_path: str, + data_resolution: List[int], + num_input_frames: int, + num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT, +) -> dict: + r""" + Function to read input videos. + Args: + batch_input_path (str): Path to JSONL file containing visual input paths + data_resolution (list[int]): Data resolution + Returns: + Dict containing input videos + """ + # Read visual inputs from JSONL + flist = [] + with open(batch_input_path, "r") as f: + for line in f: + data = json.loads(line.strip()) + flist.append(data["visual_input"]) + return load_videos_from_list( + flist, data_resolution=data_resolution, num_input_frames=num_input_frames, num_total_frames=num_total_frames + ) + + +def read_input_video( + input_path: str, + data_resolution: List[int], + num_input_frames: int, + num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT, +) -> dict: + """ + Function to read input video. + Args: + input_path (str): Path to input video + data_resolution (List[int]): Data resolution + num_input_frames (int): Number of frames in context + Returns: + Dict containing input video + """ + flist = [input_path] + return load_videos_from_list( + flist, data_resolution=data_resolution, num_input_frames=num_input_frames, num_total_frames=num_total_frames + ) + + +def load_videos_from_list( + flist: List[str], + data_resolution: List[int], + num_input_frames: int, + num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT, +) -> dict: + """ + Function to load videos from a list of video paths. + Args: + flist (List[str]): List of video paths + data_resolution (List[int]): Data resolution + num_input_frames (int): Number of frames in context + Returns: + Dict containing input videos + """ + all_videos = dict() + + for video_path in flist: + ext = os.path.splitext(video_path)[-1] + if ext in _VIDEO_EXTENSIONS: + video, _, _ = torchvision.io.read_video(video_path, pts_unit="sec") + video = video.float() / 255.0 + video = video * 2 - 1 + + # Resize the videos to the required dimension + nframes_in_video = video.shape[0] + if nframes_in_video < num_input_frames: + fname = os.path.basename(video_path) + log.warning( + f"Video {fname} has {nframes_in_video} frames, less than the requried {num_input_frames} frames. Skipping." + ) + continue + + video = video[-num_input_frames:, :, :, :] + + # Pad the video to NUM_TOTAL_FRAMES (because the tokenizer expects inputs of NUM_TOTAL_FRAMES) + video = torch.cat( + (video, video[-1, :, :, :].unsqueeze(0).repeat(num_total_frames - num_input_frames, 1, 1, 1)), + dim=0, + ) + + video = video.permute(0, 3, 1, 2) + + log.debug( + f"Resizing input video of shape ({video.shape[2]}, {video.shape[3]}) -> ({data_resolution[0]}, {data_resolution[1]})" + ) + video = resize_input(video, data_resolution) + + fname = os.path.basename(video_path) + all_videos[fname] = video.transpose(0, 1).unsqueeze(0) + + return all_videos + + +def load_vision_input( + input_type: str, + batch_input_path: str, + input_image_or_video_path: str, + data_resolution: List[int], + num_input_frames: int, + num_total_frames: int = NUM_TOTAL_FRAMES_DEFAULT, +): + """ + Function to load vision input. + Note: We pad the frames of the input image/video to NUM_TOTAL_FRAMES here, and feed the padded video tensors to the video tokenizer to obtain tokens. The tokens will be truncated based on num_input_frames when feeding to the autoregressive model. + Args: + input_type (str): Type of input + batch_input_path (str): Folder containing input images or videos + input_image_or_video_path (str): Path to input image or video + data_resolution (List[int]): Data resolution + num_input_frames (int): Number of frames in context + Returns: + Dict containing input videos + """ + if batch_input_path: + log.info(f"Reading batch inputs from path: {batch_input_path}") + if input_type == "image" or input_type == "text_and_image": + input_videos = read_input_images( + batch_input_path, data_resolution=data_resolution, num_total_frames=num_total_frames + ) + elif input_type == "video" or input_type == "text_and_video": + input_videos = read_input_videos( + batch_input_path, + data_resolution=data_resolution, + num_input_frames=num_input_frames, + num_total_frames=num_total_frames, + ) + else: + raise ValueError(f"Invalid input type {input_type}") + else: + if input_type == "image" or input_type == "text_and_image": + input_videos = read_input_image( + input_image_or_video_path, data_resolution=data_resolution, num_total_frames=num_total_frames + ) + elif input_type == "video" or input_type == "text_and_video": + input_videos = read_input_video( + input_image_or_video_path, + data_resolution=data_resolution, + num_input_frames=num_input_frames, + num_total_frames=num_total_frames, + ) + else: + raise ValueError(f"Invalid input type {input_type}") + return input_videos + + +def prepare_video_batch_for_saving(video_batch: List[torch.Tensor]) -> List[np.ndarray]: + """ + Function to convert output tensors to numpy format for saving. + Args: + video_batch (List[torch.Tensor]): List of output tensors + Returns: + List of numpy arrays + """ + return [(video * 255).to(torch.uint8).permute(1, 2, 3, 0).cpu().numpy() for video in video_batch] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/misc.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/misc.py new file mode 100644 index 00000000..602ea1cb --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/misc.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from omegaconf import DictConfig, OmegaConf + + +class CustomSimpleNamespace: + """ + A simple namespace class that supports both attribute-style and dictionary-style access. + """ + + def __init__(self, d): + self._d = d + + def __getattr__(self, attr): + # Attribute-style access: config.key + try: + return self._d[attr] + except KeyError: + raise AttributeError(f"'CustomSimpleNamespace' object has no attribute '{attr}'") + + def __getitem__(self, key): + # Dictionary-style access: config['key'] + return self._d[key] + + +def maybe_convert_to_namespace(config): + """ + This function cast a OmegaConf's DictConfig or a standard dict to CustomSimpleNamespace, which supports both + attribute-style and dictionary-style access. + Note: We need to convert OmegaConf's DictConfig since it is not compatible with torch.compile. + """ + # If input is OmegaConf's DictConfig, convert to a standard dict + if isinstance(config, DictConfig): + config = OmegaConf.to_container(config, resolve=True) + + if isinstance(config, dict): + return CustomSimpleNamespace(config) + else: + return config diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/sampling.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/sampling.py new file mode 100644 index 00000000..369d7c31 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/autoregressive/utils/sampling.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional, Tuple + +import torch + +from cosmos1.models.autoregressive.networks.transformer import Transformer + + +def sample_top_p(logits, temperature, top_p, return_probs: bool = False): + """ + Perform top-p (nucleus) sampling on a probability distribution. + + Args: + logits (torch.Tensor): Logits of the probability distribution. + temperature (float): Temperature for sampling. + top_p (float): Probability threshold for top-p sampling. + + Returns: + torch.Tensor: Sampled token indices. + + Note: + Top-p sampling selects the smallest set of tokens whose cumulative probability mass + exceeds the threshold p. The distribution is renormalized based on the selected tokens. + """ + probs = torch.softmax(logits[:, -1, :] / temperature, dim=-1) + # Sort the probabilities in descending order and get their indices. + probs_sort, probs_idx = torch.sort(probs, dim=-1, descending=True) + # Compute the cumulative sum of the sorted probabilities. + probs_sum = torch.cumsum(probs_sort, dim=-1) + # Create a mask where the cumulative probability exceeds the threshold p. + mask = probs_sum - probs_sort > top_p + # Set the probabilities that exceed the threshold to 0. + probs_sort[mask] = 0.0 + # Renormalize the remaining probabilities so they sum to 1. + probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True)) + # Sample from the renormalized probability distribution. + # next_token = torch.multinomial(probs_sort, num_samples=1) + next_token = multinomial_sample_one_no_sync(probs_sort, dtype=torch.int64) + # Gather the indices of the sampled tokens. + next_token = torch.gather(probs_idx, -1, next_token) + if return_probs: + # Initialize a tensor for unsorted probabilities + probs_unsorted = torch.zeros_like(probs_sort) + # Scatter the sorted probabilities back to their original order + probs_unsorted.scatter_(-1, probs_idx, probs_sort) + else: + probs_unsorted = None + return next_token, probs_unsorted + + +def multinomial_sample_one_no_sync(probs_sort, dtype=torch.int): + """ + Multinomial sampling without a cuda synchronization. + Source: https://github.com/pytorch-labs/gpt-fast/blob/main/generate.py + """ + q = torch.empty_like(probs_sort).exponential_(1) + return torch.argmax(probs_sort / q, dim=-1, keepdim=True).to(dtype=dtype) + + +def logits_to_probs( + logits, + temperature: float = 1.0, + top_k: Optional[int] = None, +): + logits = logits / max(temperature, 1e-5) + + if top_k is not None: + v, _ = torch.topk(logits, min(top_k, logits.size(-1))) + pivot = v.select(-1, -1).unsqueeze(-1) + logits = torch.where(logits < pivot, -float("Inf"), logits) + probs = torch.nn.functional.softmax(logits, dim=-1) + return probs + + +def sample_top_k(logits, temperature: float = 1.0, top_k: Optional[int] = None): + """ + Sample from the logits using top-k sampling. + Source: https://github.com/pytorch-labs/gpt-fast/blob/main/generate.py + """ + # logits: [batch_size, seq_len, vocab_size] + if temperature == 0.0: + idx_next = torch.argmax(logits[:, -1, :], dim=-1, keepdim=True) + probs = None + else: + probs = logits_to_probs(logits[:, -1, :], temperature, top_k) + idx_next = multinomial_sample_one_no_sync(probs) + return idx_next, probs + + +def prefill( + model: Transformer, + input_pos: torch.Tensor, + tokens: torch.Tensor = None, + token_embeddings: torch.Tensor = None, + temperature: float = 1.0, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + **kwargs, +) -> torch.Tensor: + logits = model(tokens=tokens, token_embeddings=token_embeddings, input_pos=input_pos, **kwargs) + # Only top-p or top-k can be provided + assert top_p is None or top_k is None, ( + "Only one of top-p or top-k can be provided, got top-p={top_p} and top-k={top_k}" + ) + if top_p is not None: + return sample_top_p(logits, temperature=temperature, top_p=top_p)[0] + else: + return sample_top_k(logits, temperature=temperature, top_k=top_k)[0] + + +def decode_one_token( + model: Transformer, + tokens: torch.Tensor, + input_pos: torch.Tensor, + temperature: float = 1.0, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + **kwargs, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Decode a single token from the autoregressive model. + """ + logits = model(tokens=tokens, input_pos=input_pos, **kwargs) + if top_p is not None: + return sample_top_p(logits, temperature=temperature, top_p=top_p) + else: + return sample_top_k(logits, temperature=temperature, top_k=top_k) + + +def decode_n_tokens( + model: Transformer, + cur_token: torch.Tensor, + input_pos: torch.Tensor, + num_new_tokens: int, + stop_tokens: torch.Tensor = None, + temperature: float = 1.0, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + return_probs: bool = False, + decode_one_token_function=decode_one_token, + **kwargs, +): + """ + Decode n tokens from the autoregressive model. + Adapted from https://github.com/pytorch-labs/gpt-fast/blob/main/generate.py + """ + new_tokens, new_probs = [], [] + batch_size = cur_token.shape[0] + assert top_p is None or top_k is None, ( + "Only one of top-p or top-k can be provided, got top-p={top_p} and top-k={top_k}" + ) + if stop_tokens is not None: + # Indicator for whether the EOS token (stop token) has been reached for each sample in the batch + eos_reached = torch.tensor([False] * batch_size, device="cuda") + for t in range(num_new_tokens): + with torch.backends.cuda.sdp_kernel( + enable_flash=False, enable_mem_efficient=False, enable_math=True + ): # Actually better for Inductor to codegen attention here + next_token, next_prob = decode_one_token_function( + model, + tokens=cur_token, + input_pos=input_pos, + temperature=temperature, + top_k=top_k, + top_p=top_p, + **kwargs, + ) + input_pos += 1 + if stop_tokens is not None and len(stop_tokens) > 0: + eos_reached = eos_reached | (torch.isin(next_token, stop_tokens)) + if eos_reached.all(): + break + new_tokens.append(next_token.clone()) + if return_probs: + new_probs.append(next_prob.clone()) + cur_token = next_token.clone() + + if return_probs: + return new_tokens, new_probs + else: + return new_tokens diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/common/base_world_generation_pipeline.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/common/base_world_generation_pipeline.py new file mode 100644 index 00000000..77addaa5 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/common/base_world_generation_pipeline.py @@ -0,0 +1,359 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import os +from abc import ABC +from typing import Any + +import numpy as np +import torch +from cosmos1.models.common.t5_text_encoder import CosmosT5TextEncoder +from cosmos1.models.guardrail.common import presets as guardrail_presets + + +class BaseWorldGenerationPipeline(ABC): + def __init__( + self, + inference_type: str | None = None, + checkpoint_dir: str | None = None, + checkpoint_name: str | None = None, + has_text_input: bool = False, + offload_network: bool = False, + offload_tokenizer: bool = False, + offload_text_encoder_model: bool = False, + offload_guardrail_models: bool = False, + ): + """Initialize base world generation pipeline. + + This abstract base class provides core functionality for world generation models including: + - Model loading and initialization + - Text encoding and embedding + - Safety checks and content filtering + - Memory management through model offloading + + Args: + inference_type: The type of inference pipeline ("text2world" or "video2world") + checkpoint_dir: Root directory containing model checkpoints + checkpoint_name: Name of the specific checkpoint file to load + has_text_input: Whether the pipeline takes text input for world generation + offload_network: If True, moves main model to CPU after inference + offload_tokenizer: If True, moves tokenizer to CPU after use + offload_text_encoder_model: If True, moves T5 encoder to CPU after encoding + offload_guardrail_models: If True, moves safety models to CPU after checks + """ + self.inference_type = inference_type + self.checkpoint_dir = checkpoint_dir + self.checkpoint_name = checkpoint_name + self.guardrail_dir = "Cosmos-1.0-Guardrail" + self.has_text_input = has_text_input + + # Add offloading flags + self.offload_network = offload_network + self.offload_tokenizer = offload_tokenizer + self.offload_text_encoder_model = offload_text_encoder_model + self.offload_guardrail_models = offload_guardrail_models + + # Initialize model instances + self.text_guardrail = None + self.video_guardrail = None + self.text_encoder = None + self.model = None + + self._load_model() + + if not self.offload_text_encoder_model: + self._load_text_encoder_model() + if not self.offload_guardrail_models: + if self.has_text_input: + self._load_text_guardrail() + self._load_video_guardrail() + if not self.offload_network: + self._load_network() + if not self.offload_tokenizer: + self._load_tokenizer() + + def _load_tokenizer(self): + pass + + def _load_network(self): + pass + + def _load_model(self, checkpoint_name: str) -> Any: + """Load the world generation model from a checkpoint. + + This abstract method must be implemented by subclasses to load their specific + model architecture and weights. + + Args: + checkpoint_name: Path to the model checkpoint file + + Returns: + The loaded model instance + + Raises: + NotImplementedError: Must be implemented by subclasses + """ + pass + + def _load_text_encoder_model(self): + """Load the T5 text encoder model. + + Initializes and loads the T5 encoder model used for converting text prompts + into embeddings that condition the world generation model. + + Returns: + Loaded T5 text encoder model instance + """ + self.text_encoder = CosmosT5TextEncoder(cache_dir=self.checkpoint_dir) + + def _load_text_guardrail(self): + """Load text safety classifier models. + + Initializes models used for checking input prompts against safety policies. + Models are loaded from the specified guardrail directory. + """ + self.text_guardrail = guardrail_presets.create_text_guardrail_runner( + checkpoint_dir=os.path.join(self.checkpoint_dir, self.guardrail_dir) + ) + + def _load_video_guardrail(self): + """Load video safety classifier models. + + Initializes models used for validating generated video content against + safety policies. Models are loaded from the specified guardrail directory. + """ + self.video_guardrail = guardrail_presets.create_video_guardrail_runner( + checkpoint_dir=os.path.join(self.checkpoint_dir, self.guardrail_dir) + ) + + def _offload_network(self): + if self.model.model: + del self.model.model + self.model.model = None + gc.collect() + torch.cuda.empty_cache() + + def _offload_tokenizer(self): + if self.model.tokenizer: + del self.model.tokenizer + self.model.tokenizer = None + gc.collect() + torch.cuda.empty_cache() + + def _offload_guardrail_models(self): + """Offload safety classifier models to reduce memory usage. + + Moves safety models to CPU and clears GPU memory if they are no longer needed. + This helps manage memory when processing multiple inputs sequentially. + """ + if self.text_guardrail: + del self.text_guardrail + self.text_guardrail = None + if self.video_guardrail: + del self.video_guardrail + self.video_guardrail = None + gc.collect() + torch.cuda.empty_cache() + + def _offload_text_encoder_model(self): + """Offload T5 text encoder to reduce memory usage. + + Moves the T5 encoder to CPU and clears GPU memory after text encoding is complete. + This helps manage memory when processing multiple inputs sequentially. + """ + if self.text_encoder: + del self.text_encoder + self.text_encoder = None + gc.collect() + torch.cuda.empty_cache() + + def _run_model(self, *args: Any, **kwargs: Any) -> torch.Tensor: + """Generate world latents using the model. + + This abstract method must be implemented by subclasses to define their specific + generation process. + + Args: + *args: Variable positional arguments for model inference + **kwargs: Variable keyword arguments for model inference + + Returns: + torch.Tensor: Generated world representation tensor + """ + pass + + def _run_model_with_offload(self, *args: Any, **kwargs: Any) -> torch.Tensor: + """Generate world representation with memory management. + + Handles loading the model before inference and offloading afterward if enabled. + This helps minimize GPU memory usage during inference. + + Args: + *args: Arguments passed to _run_model + **kwargs: Keyword arguments passed to _run_model + + Returns: + np.ndarray: Generated world representation as numpy array + """ + pass + + def _run_guardrail_on_prompt(self, prompt: str) -> bool: + """Check if prompt meets safety requirements. + + Validates the input prompt against safety policies using loaded guardrail models. + + Args: + prompt: Raw text prompt to validate + + Returns: + bool: True if prompt passes all safety checks, False otherwise + """ + return guardrail_presets.run_text_guardrail(prompt, self.text_guardrail) + + def _run_guardrail_on_prompt_with_offload(self, prompt: str) -> bool: + """Check prompt safety with memory management. + + Validates prompt safety while handling model loading/offloading to manage memory. + + Args: + prompt: Raw text prompt to validate + + Returns: + bool: True if prompt passes all safety checks, False otherwise + """ + if self.offload_guardrail_models: + self._load_text_guardrail() + + is_safe = self._run_guardrail_on_prompt(prompt) + + if self.offload_guardrail_models: + self._offload_guardrail_models() + + return is_safe + + def _run_guardrail_on_video(self, video: np.ndarray) -> np.ndarray | None: + """Check if video meets safety requirements. + + Validates generated video content against safety policies using guardrail models. + + Args: + video: Video frames to validate + + Returns: + np.ndarray: Processed video if safe, None if unsafe + """ + return guardrail_presets.run_video_guardrail(video, self.video_guardrail) + + def _run_guardrail_on_video_with_offload(self, video: np.ndarray) -> np.ndarray | None: + """Check if generated video meets safety requirements. + + Args: + video: Video frames to validate + + Returns: + np.ndarray: Processed video frames if safe, None otherwise + + Note: + Guardrail models are offloaded after checks if enabled. + """ + if self.offload_guardrail_models: + self._load_video_guardrail() + + video = self._run_guardrail_on_video(video) + + if self.offload_guardrail_models: + self._offload_guardrail_models() + return video + + def _run_text_embedding_on_prompt( + self, prompts: list[str], **kwargs: Any + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + """Convert text prompts to embeddings. + + Processes text prompts into embedding tensors that condition the generation model. + + Args: + prompts: List of text prompts to encode + **kwargs: Additional arguments for text encoding + + Returns: + tuple containing: + - List of text embedding tensors for each prompt + - List of attention masks for each embedding + """ + + embeddings = [] + masks = [] + for prompt in prompts: + embedding, mask = self.text_encoder.encode_prompts( + [prompt], + **kwargs, + ) + embeddings.append(embedding) + masks.append(mask) + + return embeddings, masks + + def _run_text_embedding_on_prompt_with_offload( + self, prompts: list[str], **kwargs: Any + ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: + """Convert text prompt into embeddings using T5 encoder. + + Args: + prompt: Processed and validated text prompt + + Returns: + Text embedding tensor to condition diffusion model + + Note: + T5 model is offloaded after encoding if enabled. + """ + if self.offload_text_encoder_model: + self._load_text_encoder_model() + + embeddings, masks = self._run_text_embedding_on_prompt(prompts, **kwargs) + + if self.offload_text_encoder_model: + self._offload_text_encoder_model() + return embeddings, masks + + def _run_tokenizer_decoding(self, samples: torch.Tensor) -> np.ndarray: + """Decode model outputs into final world representation. + + This abstract method must be implemented by subclasses to convert raw model + outputs into their specific world representation format. + + Args: + samples: Raw output tensor from the generation model + + Returns: + np.ndarray: Decoded world representation + """ + pass + + def generate(self, *args: Any, **kwargs: Any): + """Generate world representation. + + This abstract method must be implemented by subclasses to convert raw model + outputs into their specific world representation format. + + Args: + *args: Variable positional arguments for model inference + **kwargs: Variable keyword arguments for model inference + """ + pass diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/common/t5_text_encoder.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/common/t5_text_encoder.py new file mode 100644 index 00000000..944f1ec9 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/common/t5_text_encoder.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Tuple, Union + +import torch +import transformers +from cosmos1.utils import log +from transformers import T5EncoderModel, T5TokenizerFast + + +transformers.logging.set_verbosity_error() + + +class CosmosT5TextEncoder(torch.nn.Module): + """Handles T5 text encoding operations.""" + + def __init__(self, model_name: str = "google-t5/t5-11b", device: str = "cuda", cache_dir: str = "~/.cache"): + """Initializes the T5 tokenizer and encoder. + + Args: + model_name: The name of the T5 model to use. + device: The device to use for computations. + """ + super().__init__() + try: + self.tokenizer = T5TokenizerFast.from_pretrained(model_name, cache_dir=cache_dir) + self.text_encoder = T5EncoderModel.from_pretrained(model_name, cache_dir=cache_dir).to(device) + except Exception as e: + log.warning( + f"Failed to load T5 model using cache_dir '{cache_dir}', falling back to default location: {e}" + ) + self.tokenizer = T5TokenizerFast.from_pretrained(model_name) + self.text_encoder = T5EncoderModel.from_pretrained(model_name).to(device) + self.text_encoder.eval() + self.device = device + + @torch.inference_mode() + def encode_prompts( + self, prompts: Union[str, List[str]], max_length: int = 512 + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Encodes text prompts into hidden state representations using a T5 encoder. + + This function tokenizes the input prompts, processes them through a T5 text encoder, + and returns the last hidden states. The encoded outputs beyond the actual sequence + length are zero-padded. All prompts in a batch are padded to max_length. + + Args: + prompts: Input text to encode. Can be a single string or a list of strings. + max_length: Maximum sequence length for tokenization and padding. Longer + sequences will be truncated. Defaults to 512. + return_mask: If True, returns the attention mask along with encoded text. + Defaults to False. + + Returns: + If return_mask is False: + torch.Tensor: Encoded text embeddings of shape (batch_size, max_length, hidden_size). + If return_mask is True: + tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - Encoded text embeddings of shape (batch_size, max_length, hidden_size) + - Attention mask of shape (batch_size, max_length) as boolean tensor + + Raises: + ValueError: If the input prompts list is empty. + + Example: + >>> encoder = CosmosT5TextEncoder() + >>> prompts = ["Hello world", "Another example"] + >>> embeddings = encoder.encode_prompts(prompts, max_length=128) + """ + if isinstance(prompts, str): + prompts = [prompts] + + if not prompts: + raise ValueError("The input prompt list is empty.") + + batch_encoding = self.tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + input_ids = batch_encoding.input_ids.to(self.device) + attn_mask = batch_encoding.attention_mask.to(self.device) + + outputs = self.text_encoder(input_ids=input_ids, attention_mask=attn_mask) + + encoded_text = outputs.last_hidden_state + lengths = attn_mask.sum(dim=1).cpu() + + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text, attn_mask diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/README.md new file mode 100644 index 00000000..1f6babc8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/README.md @@ -0,0 +1,408 @@ +# Cosmos Diffusion-based World Foundation Models + +## Table of Contents +- [Getting Started](#getting-started) + - [Set Up Docker Environment](#set-up-docker-environment) + - [Download Checkpoints](#download-checkpoints) +- [Usage](#usage) + - [Model Types](#model-types) + - [Single and Batch Generation](#single-and-batch-generation) + - [Sample Commands](#sample-commands) + - [Text2World](#text2world-text2worldpy-7b-and-14b) + - [Video2World](#video2world-video2worldpy-7b-and-14b) + - [Arguments](#arguments) + - [Common Parameters](#common-parameters) + - [Text2World Specific Parameters](#text2world-specific-parameters) + - [Video2World Specific Parameters](#video2world-specific-parameters) + - [Safety Features](#safety-features) + - [Prompting Instructions](#prompting-instructions) + +This page details the steps for using the Cosmos diffusion-based world foundation models. + +## Getting Started + +### Set Up Docker Environment + +Follow our [Installation Guide](../../../INSTALL.md) to set up the Docker environment. All commands on this page should be run inside Docker. + +### Download Checkpoints + +1. Generate a [Hugging Face](https://huggingface.co/settings/tokens) access token. Set the access token to 'Read' permission (default is 'Fine-grained'). + +2. Log in to Hugging Face with the access token: + +```bash +huggingface-cli login +``` + +3. Request access to Mistral AI's Pixtral-12B model by clicking on `Agree and access repository` on [Pixtral's Hugging Face model page](https://huggingface.co/mistralai/Pixtral-12B-2409). This step is required to use Pixtral 12B for the Video2World prompt upsampling task. + +4. Download the Cosmos model weights from [Hugging Face](https://huggingface.co/collections/nvidia/cosmos-6751e884dc10e013a0a0d8e6): + +```bash +PYTHONPATH=$(pwd) python cosmos1/scripts/download_diffusion.py --model_sizes 7B 14B --model_types Text2World Video2World +``` + +5. The downloaded files should be in the following structure: + +``` +checkpoints/ +├── Cosmos-1.0-Diffusion-7B-Text2World +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Diffusion-14B-Text2World +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Diffusion-7B-Video2World +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Diffusion-14B-Video2World +│ ├── model.pt +│ └── config.json +├── Cosmos-1.0-Tokenizer-CV8x8x8 +│ ├── decoder.jit +│ ├── encoder.jit +│ └── mean_std.pt +├── Cosmos-1.0-Prompt-Upsampler-12B-Text2World +│ ├── model.pt +│ └── config.json +├── Pixtral-12B +│ ├── model.pt +│ ├── config.json +└── Cosmos-1.0-Guardrail + ├── aegis/ + ├── blocklist/ + ├── face_blur_filter/ + └── video_content_safety_filter/ +``` + +## Usage + +### Model Types + +There are two model types available for diffusion world generation: + +1. **Text2World**: Supports world generation from text input + +* Models: `Cosmos-1.0-Diffusion-7B-Text2World` and `Cosmos-1.0-Diffusion-14B-Text2World` +* Inference script: [text2world.py](/cosmos1/models/diffusion/inference/text2world.py) + +2. **Video2World**: Supports world generation from text and image/video input + +* Models: `Cosmos-1.0-Diffusion-7B-Video2World` and `Cosmos-1.0-Diffusion-14B-Video2World` +* Inference script: [video2world.py](/cosmos1/models/diffusion/inference/video2world.py) + +### Single and Batch Generation + +We support both single and batch video generation. + +For generating a single video, `Text2World` mode requires the input argument `--prompt` (text input). `Video2World` mode requires `--input_image_or_video_path` (image/video input). Additionally for Video2World, if the prompt upsampler is disabled, a text prompt must also be provided using the `--prompt` argument. + +For generating a batch of videos, both `Text2World` and `Video2World` require `--batch_input_path` (path to a JSONL file). For `Text2World`, the JSONL file should contain one prompt per line in the following format, where each line must contain a "prompt" field: + +```json +{"prompt": "prompt1"} +{"prompt": "prompt2"} +``` + +For `Video2World`, each line in the JSONL file must contain a "visual_input" field: + +```json +{"visual_input": "path/to/video1.mp4"} +{"visual_input": "path/to/video2.mp4"} +``` + +If you disable the prompt upsampler by setting the `--disable_prompt_upsampler` flag, each line in the JSONL file will need to include both "prompt" and "visual_input" fields. + +```json +{"prompt": "prompt1", "visual_input": "path/to/video1.mp4"} +{"prompt": "prompt2", "visual_input": "path/to/video2.mp4"} +``` + +### Sample Commands + +There are two main demo scripts for diffusion world generation: `text2world.py` and `video2world.py`. Below you will find sample commands for single and batch generation, as well as commands for running with low-memory GPUs using model offloading. We also provide a memory usage table comparing different offloading strategies to help with configuration. + +#### Text2World (text2world.py): 7B and 14B + +Generates world from text input. + +##### Single Generation + +```bash +PROMPT="A sleek, humanoid robot stands in a vast warehouse filled with neatly stacked cardboard boxes on industrial shelves. \ +The robot's metallic body gleams under the bright, even lighting, highlighting its futuristic design and intricate joints. \ +A glowing blue light emanates from its chest, adding a touch of advanced technology. The background is dominated by rows of boxes, \ +suggesting a highly organized storage system. The floor is lined with wooden pallets, enhancing the industrial setting. \ +The camera remains static, capturing the robot's poised stance amidst the orderly environment, with a shallow depth of \ +field that keeps the focus on the robot while subtly blurring the background for a cinematic effect." + +# Example using 7B model +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/text2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Text2World \ + --prompt "$PROMPT" \ + --offload_prompt_upsampler \ + --video_save_name Cosmos-1.0-Diffusion-7B-Text2World + +# Example using the 7B model on low-memory GPUs with model offloading. The speed is slower if using batch generation. +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/text2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Text2World \ + --prompt "$PROMPT" \ + --video_save_name Cosmos-1.0-Diffusion-7B-Text2World_memory_efficient \ + --offload_tokenizer \ + --offload_diffusion_transformer \ + --offload_text_encoder_model \ + --offload_prompt_upsampler \ + --offload_guardrail_models + +# Example using 14B model with prompt upsampler offloading (required on H100) +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/text2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-14B-Text2World \ + --prompt "$PROMPT" \ + --video_save_name Cosmos-1.0-Diffusion-14B-Text2World \ + --offload_prompt_upsampler \ + --offload_guardrail_models +``` + +##### Batch Generation + +```bash +# Example using 7B model +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/text2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Text2World \ + --batch_input_path cosmos1/models/diffusion/assets/v1p0/batch_inputs/text2world.jsonl \ + --video_save_folder outputs/Cosmos-1.0-Diffusion-7B-Text2World \ + --offload_prompt_upsampler +``` + +##### Example Output + +Here is an example output video generated using text2world.py, using `Cosmos-1.0-Diffusion-7B-Text2World`: + + + +The upsampled prompt used to generate the video is: + +``` +In a sprawling, meticulously organized warehouse, a sleek humanoid robot stands sentinel amidst towering shelves brimming with neatly stacked cardboard boxes. The robot's metallic body, adorned with intricate joints and a glowing blue chest light, radiates an aura of advanced technology, its design a harmonious blend of functionality and futuristic elegance. The camera captures this striking figure in a static, wide shot, emphasizing its poised stance against the backdrop of industrial wooden pallets. The lighting is bright and even, casting a warm glow that accentuates the robot's form, while the shallow depth of field subtly blurs the rows of boxes, creating a cinematic depth that draws the viewer into this high-tech realm. The absence of human presence amplifies the robot's solitary vigil, inviting contemplation of its purpose within this vast, organized expanse. +``` + +If you disable the prompt upsampler by using the `--disable_prompt_upsampler` flag, the output video will be generated using the original prompt: + + + +The original prompt is: +``` +A sleek, humanoid robot stands in a vast warehouse filled with neatly stacked cardboard boxes on industrial shelves. The robot's metallic body gleams under the bright, even lighting, highlighting its futuristic design and intricate joints. A glowing blue light emanates from its chest, adding a touch of advanced technology. The background is dominated by rows of boxes, suggesting a highly organized storage system. The floor is lined with wooden pallets, enhancing the industrial setting. The camera remains static, capturing the robot's poised stance amidst the orderly environment, with a shallow depth of field that keeps the focus on the robot while subtly blurring the background for a cinematic effect. +``` + +Note that the robot face could be blurred sometimes by the guardrail in this example. + +##### Inference Time and GPU Memory Usage + +The numbers provided below may vary depending on system specs and are for reference only. + +We report the maximum observed GPU memory usage during end-to-end inference. Additionally, we offer a series of model offloading strategies to help users manage GPU memory usage effectively. + +For GPUs with limited memory (e.g., RTX 3090/4090 with 24 GB memory), we recommend fully offloading all models. For higher-end GPUs, users can select the most suitable offloading strategy considering the numbers provided below. + +| Offloading Strategy | 7B Text2World | 14B Text2World | +|-------------|---------|---------| +| Offload prompt upsampler | 74.0 GB | > 80.0 GB | +| Offload prompt upsampler & guardrails | 57.1 GB | 70.5 GB | +| Offload prompt upsampler & guardrails & T5 encoder | 38.5 GB | 51.9 GB | +| Offload prompt upsampler & guardrails & T5 encoder & tokenizer | 38.3 GB | 51.7 GB | +| Offload prompt upsampler & guardrails & T5 encoder & tokenizer & diffusion model | 24.4 GB | 39.0 GB | + +The table below presents the end-to-end inference runtime on a single H100 GPU, excluding model initialization time. + +| 7B Text2World (offload prompt upsampler) | 14B Text2World (offload prompt upsampler, guardrails) | +|---------|---------| +| ~380 seconds | ~590 seconds | + +#### Video2World (video2world.py): 7B and 14B + +Generates world from text and image/video input. + +##### Single Generation + +Note that our prompt upsampler is enabled by default for Video2World, and it will generate the prompt from the input image/video. If the prompt upsampler is disabled, you can provide a prompt manually using the `--prompt` flag. + +```bash +# Example using the 7B model +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/video2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Video2World \ + --input_image_or_video_path cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg \ + --num_input_frames 1 \ + --video_save_name Cosmos-1.0-Diffusion-7B-Video2World \ + --offload_prompt_upsampler + +# Example using the 7B model on low-memory GPUs with model offloading. The speed is slower if using batch generation. +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/video2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Video2World \ + --input_image_or_video_path cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg \ + --num_input_frames 1 \ + --video_save_name Cosmos-1.0-Diffusion-7B-Video2World_memory_efficient \ + --offload_tokenizer \ + --offload_diffusion_transformer \ + --offload_text_encoder_model \ + --offload_prompt_upsampler \ + --offload_guardrail_models + +# Example using 14B model with prompt upsampler offloading (required on H100) +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/video2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-14B-Video2World \ + --input_image_or_video_path cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg \ + --num_input_frames 1 \ + --video_save_name Cosmos-1.0-Diffusion-14B-Video2World \ + --offload_prompt_upsampler \ + --offload_guardrail_models +``` + +##### Batch Generation + +```bash +# Example using 7B model with 9 input frames +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/video2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Video2World \ + --batch_input_path cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_ps.jsonl \ + --video_save_folder outputs/Cosmos-1.0-Diffusion-7B-Video2World \ + --offload_prompt_upsampler \ + --num_input_frames 9 + +# Example using 7B model with 9 input frames without prompt upsampler, using 'prompt' field in the JSONL file +PYTHONPATH=$(pwd) python cosmos1/models/diffusion/inference/video2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-1.0-Diffusion-7B-Video2World \ + --batch_input_path cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_wo_ps.jsonl \ + --video_save_folder outputs/Cosmos-1.0-Diffusion-7B-Video2World_wo_ps \ + --disable_prompt_upsampler \ + --num_input_frames 9 +``` + +##### Example Output + +Here is an example output video generated using video2world.py, using `Cosmos-1.0-Diffusion-14B-Video2World`: + + + +The upsampled prompt (generated by the prompt upsampler) used to generate the video is: + +``` +The video depicts a long, straight highway stretching into the distance, flanked by metal guardrails. The road is divided into multiple lanes, with a few vehicles visible in the far distance. The surrounding landscape features dry, grassy fields on one side and rolling hills on the other. The sky is mostly clear with a few scattered clouds, suggesting a bright, sunny day. +``` + +##### Inference Time and GPU Memory Usage + +The numbers provided below may vary depending on system specs and are for reference only. + +| Offloading Strategy | 7B Video2World | 14B Video2World | +|----------------------------------------------------------------------------------|---------|---------| +| Offload prompt upsampler | 76.5 GB | > 80.0 GB | +| Offload prompt upsampler & guardrails | 59.9 GB | 73.3 GB | +| Offload prompt upsampler & guardrails & T5 encoder | 41.3 GB | 54.8 GB | +| Offload prompt upsampler & guardrails & T5 encoder & tokenizer | 41.1 GB | 54.5 GB | +| Offload prompt upsampler & guardrails & T5 encoder & tokenizer & diffusion model | 27.3 GB | 39.0 GB | + +The following table shows the end-to-end inference runtime on a single H100 GPU, excluding model initialization time: + +| 7B Video2World (offload prompt upsampler) | 14B Video2World (offload prompt upsampler, guardrails) | +|---------|---------| +| ~383 seconds | ~593 seconds | + +### Arguments + +#### Common Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--checkpoint_dir` | Directory containing model weights | "checkpoints" | +| `--tokenizer_dir` | Directory containing tokenizer weights | "Cosmos-1.0-Tokenizer-CV8x8x8" | +| `--video_save_name` | Output video filename for single video generation | "output" | +| `--video_save_folder` | Output directory for batch video generation | "outputs/" | +| `--prompt` | Text prompt for single video generation. Required for single video generation. | None | +| `--batch_input_path` | Path to JSONL file for batch video generation. Required for batch video generation. | None | +| `--negative_prompt` | Negative prompt for improved quality | "The video captures a series of frames showing ugly scenes..." | +| `--num_steps` | Number of diffusion sampling steps | 35 | +| `--guidance` | CFG guidance scale | 7.0 | +| `--num_video_frames` | Number of frames to generate | 121 | +| `--height` | Output video height | 704 | +| `--width` | Output video width | 1280 | +| `--fps` | Frames per second | 24 | +| `--seed` | Random seed | 1 | +| `--disable_prompt_upsampler` | Disable automatic prompt enhancement | False | +| `--offload_diffusion_transformer` | Offload DiT model after inference, used for low-memory GPUs | False | +| `--offload_tokenizer` | Offload VAE model after inference, used for low-memory GPUs | False | +| `--offload_text_encoder_model` | Offload text encoder after inference, used for low-memory GPUs | False | +| `--offload_prompt_upsampler` | Offload prompt upsampler after inference, used for low-memory GPUs | False | +| `--offload_guardrail_models` | Offload guardrail models after inference, used for low-memory GPUs | False | + +Note: we support various aspect ratios, including 1:1 (960x960 for height and width), 4:3 (960x704), 3:4 (704x960), 16:9 (1280x704), and 9:16 (704x1280). The frame rate is also adjustable within a range of 12 to 40 fps. The current version of the model only supports 121 frames. + +#### Text2World Specific Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--diffusion_transformer_dir` | Directory containing DiT weights | "Cosmos-1.0-Diffusion-7B-Text2World" | +| `--prompt_upsampler_dir` | Directory containing prompt upsampler weights | "Cosmos-1.0-Prompt-Upsampler-12B-Text2World" | +| `--word_limit_to_skip_upsampler` | Skip prompt upsampler for better robustness if the number of words in the prompt is greater than this value | 250 | +#### Video2World Specific Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `--diffusion_transformer_dir` | Directory containing DiT weights | "Cosmos-1.0-Diffusion-7B-Video2World" | +| `--prompt_upsampler_dir` | Directory containing prompt upsampler weights | "Pixtral-12B" | +| `--input_image_or_video_path` | Input video/image path for single video generation. Required for single video generation. | None | +| `--num_input_frames` | Number of video frames (1 or 9) | 1 | + +### Safety Features + +The model uses a built-in safety guardrail system that cannot be disabled. Generating human faces is not allowed and will be blurred by the guardrail. + +For more information, check out the [Cosmos Guardrail Documentation](../guardrail/README.md). + +### Prompting Instructions + +The input prompt is the most important parameter under the user's control when interacting with the model. Providing rich and descriptive prompts can positively impact the output quality of the model, whereas short and poorly detailed prompts can lead to subpar video generation. Here are some recommendations to keep in mind when crafting text prompts for the model: + +1. **Describe a single, captivating scene**: Focus on a single scene to prevent the model from generating videos with unnecessary shot changes. +2. **Limit camera control instructions**: The model doesn't handle prompts involving camera control well, as this feature is still under development. +3. **Prompt upsampler limitations**: The current version of the prompt upsampler may sometimes deviate from the original intent of your prompt, adding unwanted details. If this happens, you can disable the upsampler with the --disable_prompt_upsampler flag and edit your prompt manually. We recommend using prompts of around 120 words for optimal quality. + +#### Cosmos-1.0-Prompt-Upsampler + +The prompt upsampler automatically expands brief prompts into more detailed descriptions (Text2World) or generates detailed prompts based on input images (Video2World). + +##### Text2World + +When enabled (default), the upsampler will: + +1. Take your input prompt +2. Process it through a finetuned Mistral model to generate a more detailed description +3. Use the expanded description for video generation + +This can help generate better quality videos by providing more detailed context to the video generation model. To disable this feature, use the `--disable_prompt_upsampler` flag. + +##### Video2World + +When enabled (default), the upsampler will: + +1. Take your input image or video +2. Process it through a Pixtral model to generate a detailed description +3. Use the generated description for video generation + +Please note that the Video2World prompt upsampler does not consider any user-provided text prompt. To disable this feature, use the `--disable_prompt_upsampler` flag. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo/text2world_example_after_finetune.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo/text2world_example_after_finetune.mp4 new file mode 100644 index 00000000..58926f58 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo/text2world_example_after_finetune.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo_diffusion_example_data/output_Digit_Lift_movie.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo_diffusion_example_data/output_Digit_Lift_movie.mp4 new file mode 100644 index 00000000..b8b0a991 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo_diffusion_example_data/output_Digit_Lift_movie.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo_diffusion_example_data/output_digit_lift.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo_diffusion_example_data/output_digit_lift.mp4 new file mode 100644 index 00000000..bcdcf3a2 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/nemo_diffusion_example_data/output_digit_lift.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/text2world.jsonl b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/text2world.jsonl new file mode 100644 index 00000000..38fc3aec --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/text2world.jsonl @@ -0,0 +1,2 @@ +{"prompt": "A tugger train glides smoothly through the factory floor. Its trailers are filled with a mix of neatly arranged crates and humanoid robots, each securely positioned and ready for delivery. The robots, sleek and functional in design, sit upright alongside the crates, emphasizing their integration into the production process. The train moves along a marked path, highlighting the organized and efficient flow of materials and automation in the bustling environment. The crates appear carefully packed, and the robots\u2019 purposeful placement suggests their importance in streamlining operations."} +{"prompt": "A sleek, humanoid robot stands in a vast warehouse filled with neatly stacked cardboard boxes on industrial shelves. The robot's metallic body gleams under the bright, even lighting, highlighting its futuristic design and intricate joints. A glowing blue light emanates from its chest, adding a touch of advanced technology. The background is dominated by rows of boxes, suggesting a highly organized storage system. The floor is lined with wooden pallets, enhancing the industrial setting. The camera remains static, capturing the robot's poised stance amidst the orderly environment, with a shallow depth of field that keeps the focus on the robot while subtly blurring the background for a cinematic effect."} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_ps.jsonl b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_ps.jsonl new file mode 100644 index 00000000..b378bd80 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_ps.jsonl @@ -0,0 +1,2 @@ +{"visual_input": "cosmos1/models/diffusion/assets/v1p0/video2world_input1.mp4"} +{"visual_input": "cosmos1/models/diffusion/assets/v1p0/video2world_input2.mp4"} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_wo_ps.jsonl b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_wo_ps.jsonl new file mode 100644 index 00000000..80adecf4 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/batch_inputs/video2world_wo_ps.jsonl @@ -0,0 +1,2 @@ +{"prompt": "The video depicts a vehicle driving along a sandy beach, leaving a trail of sand in its wake. The car is moving parallel to the shoreline, with the ocean waves gently lapping at the sand. The scene captures the essence of a serene coastal drive, emphasizing the interaction between the vehicle and the natural environment.", "visual_input": "cosmos1/models/diffusion/assets/v1p0/video2world_input1.mp4"} +{"prompt": "The video depicts a winding mountain road covered in snow, with a single vehicle traveling along it. The road is flanked by steep, rocky cliffs and sparse vegetation. The landscape is characterized by rugged terrain and a river visible in the distance. The scene captures the solitude and beauty of a winter drive through a mountainous region.", "visual_input": "cosmos1/models/diffusion/assets/v1p0/video2world_input2.mp4"} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/text2world_example.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/text2world_example.mp4 new file mode 100644 index 00000000..bb0b15ce Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/text2world_example.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/text2world_example2.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/text2world_example2.mp4 new file mode 100644 index 00000000..c593fb76 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/text2world_example2.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_example.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_example.mp4 new file mode 100644 index 00000000..3f3d51a7 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_example.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg new file mode 100644 index 00000000..108b718d Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input1.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input1.mp4 new file mode 100755 index 00000000..54e752e5 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input1.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input2.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input2.mp4 new file mode 100755 index 00000000..adaffe21 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/assets/v1p0/video2world_input2.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/conditioner.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/conditioner.py new file mode 100644 index 00000000..7d84fcbd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/conditioner.py @@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import copy +from abc import ABC, abstractmethod +from collections import defaultdict +from dataclasses import dataclass, fields +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from cosmos1.models.diffusion.diffusion.functional.batch_ops import batch_mul +from cosmos1.utils import log +from cosmos1.utils.lazy_config import instantiate + + +class BaseConditionEntry(nn.Module): + def __init__(self): + super().__init__() + + self._dropout_rate = None + self._input_key = None + self._return_dict = False + + @property + def dropout_rate(self) -> Union[float, torch.Tensor]: + return self._dropout_rate + + @property + def input_key(self) -> str: + return self._input_key + + @property + def is_return_dict(self) -> bool: + return self._return_dict + + @dropout_rate.setter + def dropout_rate(self, value: Union[float, torch.Tensor]): + self._dropout_rate = value + + @input_key.setter + def input_key(self, value: str): + self._input_key = value + + @is_return_dict.setter + def is_return_dict(self, value: bool): + self._return_dict = value + + @dropout_rate.deleter + def dropout_rate(self): + del self._dropout_rate + + @input_key.deleter + def input_key(self): + del self._input_key + + @is_return_dict.deleter + def is_return_dict(self): + del self._return_dict + + def random_dropout_input( + self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None, key: Optional[str] = None + ) -> torch.Tensor: + del key + dropout_rate = dropout_rate if dropout_rate is not None else self.dropout_rate + return batch_mul( + torch.bernoulli((1.0 - dropout_rate) * torch.ones(in_tensor.shape[0])).type_as(in_tensor), + in_tensor, + ) + + def summary(self) -> str: + pass + + +class DataType(Enum): + IMAGE = "image" + VIDEO = "video" + + +class TextAttr(BaseConditionEntry): + def __init__(self): + super().__init__() + + def forward(self, token: torch.Tensor, mask: torch.Tensor): + return {"crossattn_emb": token, "crossattn_mask": mask} + + def random_dropout_input( + self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None, key: Optional[str] = None + ) -> torch.Tensor: + if key is not None and "mask" in key: + return in_tensor + return super().random_dropout_input(in_tensor, dropout_rate, key) + + +@dataclass +class BaseVideoCondition: + crossattn_emb: torch.Tensor + crossattn_mask: torch.Tensor + data_type: DataType = DataType.VIDEO + padding_mask: Optional[torch.Tensor] = None + fps: Optional[torch.Tensor] = None + num_frames: Optional[torch.Tensor] = None + image_size: Optional[torch.Tensor] = None + scalar_feature: Optional[torch.Tensor] = None + frame_repeat: Optional[torch.Tensor] = None + + def to_dict(self) -> Dict[str, Optional[torch.Tensor]]: + return {f.name: getattr(self, f.name) for f in fields(self)} + + +@dataclass +class VideoExtendCondition(BaseVideoCondition): + video_cond_bool: Optional[torch.Tensor] = None # whether or not it conditioned on video + gt_latent: Optional[torch.Tensor] = None + condition_video_indicator: Optional[torch.Tensor] = None # 1 for condition region + + # condition_video_input_mask will concat to the input of network, along channel dim; + # Will be concat with the input tensor + condition_video_input_mask: Optional[torch.Tensor] = None + # condition_video_augment_sigma: (B, T) tensor of sigma value for the conditional input augmentation, only valid when apply_corruption_to_condition_region is "noise_with_sigma" or "noise_with_sigma_fixed" + condition_video_augment_sigma: Optional[torch.Tensor] = None + + +class GeneralConditioner(nn.Module, ABC): + """ + An abstract module designed to handle various embedding models with conditional and + unconditional configurations. This abstract base class initializes and manages a collection + of embedders that can dynamically adjust their dropout rates based on conditioning. + + Attributes: + KEY2DIM (dict): A mapping from output keys to dimensions used for concatenation. + embedders (nn.ModuleDict): A dictionary containing all embedded models initialized and + configured based on the provided configurations. + + Parameters: + emb_models (Union[List, Any]): A dictionary where keys are embedder names and values + are configurations for initializing the embedders. + + """ + + KEY2DIM = {"crossattn_emb": 1, "crossattn_mask": 1} + + def __init__(self, **emb_models: Union[List, Any]): + super().__init__() + self.embedders = nn.ModuleDict() + for n, (emb_name, embconfig) in enumerate(emb_models.items()): + embedder = instantiate(embconfig.obj) + assert isinstance(embedder, BaseConditionEntry), ( + f"embedder model {embedder.__class__.__name__} has to inherit from AbstractEmbModel" + ) + embedder.dropout_rate = getattr(embconfig, "dropout_rate", 0.0) + + if hasattr(embconfig, "input_key"): + embedder.input_key = embconfig.input_key + elif hasattr(embconfig, "input_keys"): + embedder.input_keys = embconfig.input_keys + else: + raise KeyError(f"need either 'input_key' or 'input_keys' for embedder {embedder.__class__.__name__}") + + log.debug(f"Initialized embedder #{n}-{emb_name}: \n {embedder.summary()}") + self.embedders[emb_name] = embedder + + @abstractmethod + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> Any: + """Should be implemented in subclasses to handle conditon datatype""" + raise NotImplementedError + + def _forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> Dict: + """ + Processes the input batch through all configured embedders, applying conditional dropout rates if specified. + Output tensors for each key are concatenated along the dimensions specified in KEY2DIM. + + Parameters: + batch (Dict): The input data batch to process. + override_dropout_rate (Optional[Dict[str, float]]): Optional dictionary to override default dropout rates + per embedder key. + + Returns: + Dict: A dictionary of output tensors concatenated by specified dimensions. + + Note: + In case the network code is sensitive to the order of concatenation, you can either control the order via \ + config file or make sure the embedders return a unique key for each output. + """ + output = defaultdict(list) + if override_dropout_rate is None: + override_dropout_rate = {} + + # make sure emb_name in override_dropout_rate is valid + for emb_name in override_dropout_rate.keys(): + assert emb_name in self.embedders, f"invalid name found {emb_name}" + for emb_name, embedder in self.embedders.items(): + with torch.no_grad(): + if hasattr(embedder, "input_key") and (embedder.input_key is not None): + emb_out = embedder( + embedder.random_dropout_input( + batch[embedder.input_key], override_dropout_rate.get(emb_name, None) + ) + ) + elif hasattr(embedder, "input_keys"): + emb_out = embedder( + *[ + embedder.random_dropout_input(batch[k], override_dropout_rate.get(emb_name, None), k) + for k in embedder.input_keys + ] + ) + for k, v in emb_out.items(): + output[k].append(v) + # Concatenate the outputs + return {k: torch.cat(v, dim=self.KEY2DIM.get(k, -1)) for k, v in output.items()} + + def get_condition_uncondition( + self, + data_batch: Dict, + ) -> Tuple[Any, Any]: + """ + Processes the provided data batch to generate conditioned and unconditioned outputs. + + This method manipulates dropout rates to simulate two scenarios: + 1. All conditions applied (conditioned) + 2. Conditions removed/reduced to minimum (unconditioned) + + This method sets dropout rates to zero for the conditioned scenario to fully apply + embedders' effects. For unconditioned, it sets rates to 1 (or 0 if initial rate is + insignificant) to minimize embedder influences. + + Parameters: + data_batch (Dict): Input data batch containing all necessary information for + embedding processing. + + Returns: + Tuple[Any, Any]: A tuple containing: + - Outputs with all embedders fully applied (conditioned) + - Outputs with embedders minimized/not applied (unconditioned) + """ + cond_dropout_rates, dropout_rates = {}, {} + for emb_name, embedder in self.embedders.items(): + cond_dropout_rates[emb_name] = 0.0 + dropout_rates[emb_name] = 1.0 if embedder.dropout_rate > 1e-4 else 0.0 + + condition: Any = self(data_batch, override_dropout_rate=cond_dropout_rates) + un_condition: Any = self(data_batch, override_dropout_rate=dropout_rates) + return condition, un_condition + + def get_condition_with_negative_prompt( + self, + data_batch: Dict, + ) -> Tuple[Any, Any]: + """ + Similar functionality as get_condition_uncondition + But use negative prompts for unconditon + """ + cond_dropout_rates, uncond_dropout_rates = {}, {} + for emb_name, embedder in self.embedders.items(): + cond_dropout_rates[emb_name] = 0.0 + if isinstance(embedder, TextAttr): + uncond_dropout_rates[emb_name] = 0.0 + else: + uncond_dropout_rates[emb_name] = 1.0 if embedder.dropout_rate > 1e-4 else 0.0 + + data_batch_neg_prompt = copy.deepcopy(data_batch) + if "neg_t5_text_embeddings" in data_batch_neg_prompt: + if isinstance(data_batch_neg_prompt["neg_t5_text_embeddings"], torch.Tensor): + data_batch_neg_prompt["t5_text_embeddings"] = data_batch_neg_prompt["neg_t5_text_embeddings"] + data_batch_neg_prompt["t5_text_mask"] = data_batch_neg_prompt["neg_t5_text_mask"] + + condition: Any = self(data_batch, override_dropout_rate=cond_dropout_rates) + un_condition: Any = self(data_batch_neg_prompt, override_dropout_rate=uncond_dropout_rates) + + return condition, un_condition + + +@dataclass +class CosmosCondition: + crossattn_emb: torch.Tensor + crossattn_mask: torch.Tensor + padding_mask: Optional[torch.Tensor] = None + scalar_feature: Optional[torch.Tensor] = None + + def to_dict(self) -> Dict[str, Optional[torch.Tensor]]: + return {f.name: getattr(self, f.name) for f in fields(self)} + + +class VideoConditioner(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> BaseVideoCondition: + output = super()._forward(batch, override_dropout_rate) + return BaseVideoCondition(**output) + + +class VideoExtendConditioner(GeneralConditioner): + def forward( + self, + batch: Dict, + override_dropout_rate: Optional[Dict[str, float]] = None, + ) -> VideoExtendCondition: + output = super()._forward(batch, override_dropout_rate) + return VideoExtendCondition(**output) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/conditioner.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/conditioner.py new file mode 100644 index 00000000..f01935db --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/conditioner.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Dict, List, Optional + +import attrs +import torch +from cosmos1.models.diffusion.conditioner import BaseConditionEntry, TextAttr, VideoConditioner, VideoExtendConditioner +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict + + +@attrs.define(slots=False) +class TextConfig: + obj: LazyDict = L(TextAttr)() # No arguments + dropout_rate: float = 0.2 + input_keys: List[str] = attrs.field(factory=lambda: ["t5_text_embeddings", "t5_text_mask"]) + + +class BooleanFlag(BaseConditionEntry): + def __init__(self, output_key: Optional[str] = None): + super().__init__() + self.output_key = output_key + + def forward(self, *args, **kwargs) -> Dict[str, torch.Tensor]: + del args, kwargs + key = self.output_key if self.output_key else self.input_key + return {key: self.flag} + + def random_dropout_input( + self, in_tensor: torch.Tensor, dropout_rate: Optional[float] = None, key: Optional[str] = None + ) -> torch.Tensor: + del key + dropout_rate = dropout_rate if dropout_rate is not None else self.dropout_rate + self.flag = torch.bernoulli((1.0 - dropout_rate) * torch.ones(1)).bool().to(device=in_tensor.device) + return in_tensor + + +class ReMapkey(BaseConditionEntry): + def __init__(self, output_key: Optional[str] = None, dtype: Optional[str] = None): + super().__init__() + self.output_key = output_key + self.dtype = { + None: None, + "float": torch.float32, + "bfloat16": torch.bfloat16, + "half": torch.float16, + "float16": torch.float16, + "int": torch.int32, + "long": torch.int64, + }[dtype] + + def forward(self, element: torch.Tensor) -> Dict[str, torch.Tensor]: + key = self.output_key if self.output_key else self.input_key + if isinstance(element, torch.Tensor): + element = element.to(dtype=self.dtype) + return {key: element} + + +class FrameRepeatAttr(BaseConditionEntry): + def __init__(self): + super().__init__() + + def forward(self, frame_repeat: torch.Tensor) -> Dict[str, torch.Tensor]: + return { + "frame_repeat": frame_repeat / 10.0, + } + + def details(self) -> str: + return "Frame repeat, Output key: [frame_repeat]" + + +@attrs.define(slots=False) +class FPSConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `fps`. + """ + + obj: LazyDict = L(ReMapkey)(output_key="fps", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "fps" + + +@attrs.define(slots=False) +class PaddingMaskConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `padding_mask`. + """ + + obj: LazyDict = L(ReMapkey)(output_key="padding_mask", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "padding_mask" + + +@attrs.define(slots=False) +class ImageSizeConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `image_size`. + """ + + obj: LazyDict = L(ReMapkey)(output_key="image_size", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "image_size" + + +@attrs.define(slots=False) +class NumFramesConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `num_frames`. + """ + + obj: LazyDict = L(ReMapkey)(output_key="num_frames", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "num_frames" + + +@attrs.define(slots=False) +class FrameRepeatConfig: + """ + Remap and process key from the input dictionary to the output dictionary. For `frame_repeat`. + """ + + obj: LazyDict = L(FrameRepeatAttr)() + dropout_rate: float = 0.0 + input_key: str = "frame_repeat" + + +@attrs.define(slots=False) +class VideoCondBoolConfig: + obj: LazyDict = L(BooleanFlag)(output_key="video_cond_bool") + dropout_rate: float = 0.2 + input_key: str = "fps" # This is a placeholder, we never use this value + # Config below are for long video generation only + + # Sample PPP... from IPPP... sequence + sample_tokens_start_from_p_or_i: bool = False + + +@attrs.define(slots=False) +class LatentConditionConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `latent condition`. + """ + + obj: LazyDict = L(ReMapkey)(output_key="latent_condition", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "latent_condition" + + +@attrs.define(slots=False) +class LatentConditionSigmaConfig: + """ + Remap the key from the input dictionary to the output dictionary. For `latent condition`. + """ + + obj: LazyDict = L(ReMapkey)(output_key="latent_condition_sigma", dtype=None) + dropout_rate: float = 0.0 + input_key: str = "latent_condition_sigma" + + +BaseVideoConditionerConfig: LazyDict = L(VideoConditioner)( + text=TextConfig(), +) + +VideoConditionerFpsSizePaddingConfig: LazyDict = L(VideoConditioner)( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), +) + +VideoExtendConditionerConfig: LazyDict = L(VideoExtendConditioner)( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), +) + +VideoConditionerFpsSizePaddingFrameRepeatConfig: LazyDict = L(VideoConditioner)( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + frame_repeat=FrameRepeatConfig(), +) + +VideoExtendConditionerFrameRepeatConfig: LazyDict = L(VideoExtendConditioner)( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), + frame_repeat=FrameRepeatConfig(), +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/model.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/model.py new file mode 100644 index 00000000..6654a618 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/model.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import List + +import attrs +from cosmos1.utils.lazy_config import LazyDict + + +@attrs.define(slots=False) +class DefaultModelConfig: + tokenizer: LazyDict = None + conditioner: LazyDict = None + net: LazyDict = None + sigma_data: float = 0.5 + precision: str = "bfloat16" + input_data_key: str = "video" # key to fetch input data from data_batch + latent_shape: List[int] = [16, 24, 44, 80] # 24 corresponig to 136 frames + + +@attrs.define(slots=False) +class LatentDiffusionDecoderModelConfig(DefaultModelConfig): + tokenizer_corruptor: LazyDict = None + latent_corruptor: LazyDict = None + pixel_corruptor: LazyDict = None + diffusion_decoder_cond_sigma_low: float = None + diffusion_decoder_cond_sigma_high: float = None + diffusion_decoder_corrupt_prob: float = None + condition_on_tokenizer_corruptor_token: bool = False + + +@attrs.define(slots=False) +class MultiCameraConfig(DefaultModelConfig): + n_cameras: int = 4 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/net.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/net.py new file mode 100644 index 00000000..69ebc267 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/net.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from cosmos1.models.diffusion.networks.general_dit import GeneralDIT +from cosmos1.models.diffusion.networks.general_dit_multi_camera import MultiCameraGeneralDIT +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict + + +FADITV2Config: LazyDict = L(GeneralDIT)( + max_img_h=240, + max_img_w=240, + max_frames=128, + in_channels=16, + out_channels=16, + patch_spatial=2, + patch_temporal=1, + model_channels=4096, + block_config="FA-CA-MLP", + num_blocks=28, + num_heads=32, + concat_padding_mask=True, + pos_emb_cls="rope3d", + pos_emb_learnable=False, + pos_emb_interpolation="crop", + block_x_format="THWBD", + affline_emb_norm=True, + use_adaln_lora=True, + adaln_lora_dim=256, +) + + +FADITV2_14B_Config = copy.deepcopy(FADITV2Config) +FADITV2_14B_Config.model_channels = 5120 +FADITV2_14B_Config.num_heads = 40 +FADITV2_14B_Config.num_blocks = 36 + + +FADITV2_MultiCam_Config: LazyDict = L(MultiCameraGeneralDIT)( + max_img_h=240, + max_img_w=240, + max_frames=128, + in_channels=16, + out_channels=16, + patch_spatial=2, + patch_temporal=1, + model_channels=4096, + block_config="FA-CA-MLP", + num_blocks=28, + num_heads=32, + concat_padding_mask=True, + pos_emb_cls="rope3d", + pos_emb_learnable=False, + pos_emb_interpolation="crop", + block_x_format="THWBD", + affline_emb_norm=True, + use_adaln_lora=True, + adaln_lora_dim=256, + n_cameras=6, + camera_condition_dim=6, + add_repeat_frame_embedding=True, + extra_per_block_abs_pos_emb=True, + rope_h_extrapolation_ratio=1.0, + rope_w_extrapolation_ratio=1.0, + rope_t_extrapolation_ratio=1.0, + extra_per_block_abs_pos_emb_type="sincos", +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/tokenizer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/tokenizer.py new file mode 100644 index 00000000..322d1a6c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/base/tokenizer.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import omegaconf +from cosmos1.models.diffusion.module.pretrained_vae import JITVAE, JointImageVideoSharedJITTokenizer, VideoJITTokenizer +from cosmos1.utils.lazy_config import LazyCall as L + + +TOKENIZER_OPTIONS = {} + + +def tokenizer_register(key): + def decorator(func): + TOKENIZER_OPTIONS[key] = func + return func + + return decorator + + +@tokenizer_register("cosmos_diffusion_tokenizer_comp8x8x8") +def get_cosmos_diffusion_tokenizer_comp8x8x8(resolution: str, chunk_duration: int) -> omegaconf.dictconfig.DictConfig: + assert resolution in ["720"] + + pixel_chunk_duration = chunk_duration + temporal_compression_factor = 8 + spatial_compression_factor = 8 + + return L(JointImageVideoSharedJITTokenizer)( + video_vae=L(VideoJITTokenizer)( + name="cosmos_1_0_diffusion_tokenizer", + latent_ch=16, + is_bf16=True, + pixel_chunk_duration=pixel_chunk_duration, + temporal_compression_factor=temporal_compression_factor, + spatial_compression_factor=spatial_compression_factor, + spatial_resolution=resolution, + ), + image_vae=L(JITVAE)( + name="cosmos_1_0_diffusion_tokenizer", + latent_ch=16, + is_image=False, + is_bf16=True, + ), + name="cosmos_1_0_diffusion_tokenizer", + latent_ch=16, + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/config.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/config.py new file mode 100644 index 00000000..887847ac --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/config.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, List + +import attrs +from cosmos1.models.diffusion.config.base.model import DefaultModelConfig +from cosmos1.models.diffusion.config.registry import register_configs +from cosmos1.utils import config +from cosmos1.utils.config_helper import import_all_modules_from_package + + +@attrs.define(slots=False) +class Config(config.Config): + # default config groups that will be used unless overwritten + # see config groups in registry.py + defaults: List[Any] = attrs.field( + factory=lambda: [ + "_self_", + {"net": None}, + {"conditioner": "add_fps_image_size_padding_mask"}, + {"tokenizer": "tokenizer"}, + {"experiment": None}, + ] + ) + + +def make_config(): + c = Config( + model=DefaultModelConfig(), + ) + + # Specifying values through instances of attrs + c.job.project = "cosmos_diffusion" + c.job.group = "inference" + + # Call this function to register config groups for advanced overriding. + register_configs() + + # experiment config are defined in the experiment folder + # call import_all_modules_from_package to register them + import_all_modules_from_package("cosmos1.models.diffusion.config.inference", reload=True) + return c diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/augmentors.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/augmentors.py new file mode 100644 index 00000000..722c2a21 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/augmentors.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from cosmos1.models.diffusion.config.ctrl.blurs import ( + BilateralFilterConfig, + BlurAugmentorConfig, + BlurCombinationConfig, +) +from cosmos1.utils.lazy_config import LazyCall as L + + +# predefined BilateralFilterConfig with different strength level +NoFilterConfig = L(BilateralFilterConfig)(use_random=False, d=1, sigma_color=1, sigma_space=1, iter=1) + +LowBilateralFilterConfig = L(BilateralFilterConfig)(use_random=False, d=15, sigma_color=100, sigma_space=50, iter=1) + +MediumBilateralFilterConfig = L(BilateralFilterConfig)( + use_random=False, d=30, sigma_color=150, sigma_space=100, iter=1 +) + +HighBilateralFilterConfig = L(BilateralFilterConfig)(use_random=False, d=50, sigma_color=300, sigma_space=150, iter=1) + +BilateralOnlyBlurAugmentorConfig = {} +for strength, blur_config in zip( + ["none", "very_low", "low", "medium", "high", "very_high"], + [ + NoFilterConfig, + LowBilateralFilterConfig, + LowBilateralFilterConfig, + MediumBilateralFilterConfig, + HighBilateralFilterConfig, + HighBilateralFilterConfig, + ], +): + BlurConfig = L(BlurCombinationConfig)( + blur_types=["bilateral"], + probability=1.0, + bilateral_filter=blur_config, + ) + downscale_factor = { + "none": 1, + "very_low": 1, + "low": 4, + "medium": 2, + "high": 1, + "very_high": 4, + } + BilateralOnlyBlurAugmentorConfig[strength] = L(BlurAugmentorConfig)( + blur_combinations=[BlurConfig], + downscale_factor=[downscale_factor[strength]], + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/blurs.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/blurs.py new file mode 100644 index 00000000..44e86e19 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/blurs.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional + +import attrs +from cosmos1.utils.lazy_config import LazyCall as L + + +@attrs.define +class GaussianBlurConfig: + """Configuration for Gaussian blur""" + + use_random: bool = False + # if use_random is False, then optionally define the param values + ksize: int = 25 + sigmaX: float = 12.5 + + # if use_random is True, then optionally define the range + ksize_min: int = 21 + ksize_max: int = 29 + sigmaX_min: float = 10.5 + sigmaX_max: float = 14.5 + + +LowGaussianBlurConfig = L(GaussianBlurConfig)(ksize=21, sigmaX=10.5) + + +@attrs.define +class GuidedFilterConfig: + """Configuration for Guided filter""" + + use_random: bool = False + # if use_random is False, then optionally define the param values + radius: int = 45 + eps: float = 0.15 + scale: int = 10 + + # if use_random is True, then optionally define the range + radius_min: int = 41 + radius_max: int = 49 + eps_min: float = 0.1 + eps_max: float = 0.2 + scale_min: int = 3 + scale_max: int = 18 + + +@attrs.define +class BilateralFilterConfig: + """Configuration for Bilateral filter""" + + use_random: bool = False + # if use_random is False, then optionally define the param values + d: int = 30 + sigma_color: int = 150 + sigma_space: int = 100 + iter: int = 1 + + # if use_random is True, then optionally define the range + d_min: int = 15 + d_max: int = 50 + sigma_color_min: int = 100 + sigma_color_max: int = 300 + sigma_space_min: int = 50 + sigma_space_max: int = 150 + iter_min: int = 1 + iter_max: int = 4 + + +@attrs.define +class MedianBlurConfig: + """Configuration for Median blur""" + + use_random: bool = False + # if use_random is False, then optionally define the param values + ksize: int = 11 + + # if use_random is True, then optionally define the range + ksize_min: int = 9 + ksize_max: int = 15 + + +@attrs.define +class LaplacianOfGaussianConfig: + """Configuration for LoG filter""" + + use_random: bool = False + # if use_random is False, then optionally define the param values + ksize: int = 5 + sigma: float = 1.4 + binarize: bool = False + threshold: float = 0.0 + + # if use_random is True, then optionally define the range + ksize_min: int = 3 + ksize_max: int = 7 + sigma_min: float = 0.5 + sigma_max: float = 3.0 + threshold_min: float = 10.0 + threshold_max: float = 30.0 + + +@attrs.define +class AnisotropicDiffusionConfig: + """Configuration for Anisotropic Diffusion""" + + use_random: bool = False + alpha: float = 0.25 + K: float = 0.15 + niters: int = 12 + + # if use_random is True, then optionally define the range + alpha_min: float = 0.2 + alpha_max: float = 0.3 + K_min: float = 0.1 + K_max: float = 0.2 + niters_min: int = 10 + niters_max: int = 14 + + +@attrs.define +class BlurCombinationConfig: + """Configuration for a combination of blurs with associated probability""" + + # list of choices are: ["gaussian", "guided", "bilateral", "median", "log", "anisotropic"] + # the corresponding config must be defined for each item in this blur_types list + blur_types: List[str] + probability: float + gaussian_blur: Optional[GaussianBlurConfig] = None + guided_filter: Optional[GuidedFilterConfig] = None + bilateral_filter: Optional[BilateralFilterConfig] = None + median_blur: Optional[MedianBlurConfig] = None + log: Optional[LaplacianOfGaussianConfig] = None + anisotropic_diffusion: Optional[AnisotropicDiffusionConfig] = None + + +@attrs.define +class BlurAugmentorConfig: + """Configuration for blur augmentation with multiple combinations""" + + # probabilities from the list of combinations should add up to 1.0 + blur_combinations: List[BlurCombinationConfig] = [] + downscale_factor: List[int] = [1] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/conditioner.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/conditioner.py new file mode 100644 index 00000000..d032d18f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/conditioner.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cosmos1.models.diffusion.conditioner import VideoConditionerWithCtrl +from cosmos1.models.diffusion.config.base.conditioner import ( + FPSConfig, + ImageSizeConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, + VideoCondBoolConfig, +) +from cosmos1.models.diffusion.datasets.augmentors.control_input import ( + AddControlInput, + AddControlInputCanny, + AddControlInputDepth, + AddControlInputHumanKpts, + AddControlInputIdentity, + AddControlInputMask, + AddControlInputSeg, + AddControlInputUpscale, +) +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict + + +CTRL_HINT_KEYS_SINGLE = [ + "control_input_canny", + "control_input_blur", + "control_input_depth", + "control_input_human_kpts", + "control_input_segmentation", + "control_input_mask", + "control_input_upscale", + "control_input_identity", +] + +CTRL_HINT_KEYS_MULTI = [ + "control_input_canny_blur", + "control_input_depth_segmentation", +] + +CTRL_HINT_KEYS = CTRL_HINT_KEYS_SINGLE + CTRL_HINT_KEYS_MULTI + +CTRL_HINT_KEYS_COMB = { + "control_input_canny_blur": [AddControlInputCanny, AddControlInput], + "control_input_depth_segmentation": [AddControlInputDepth, AddControlInputSeg], + "control_input_blur": [AddControlInput], + "control_input_canny": [AddControlInputCanny], + "control_input_depth": [AddControlInputDepth], + "control_input_human_kpts": [AddControlInputHumanKpts], + "control_input_segmentation": [AddControlInputSeg], + "control_input_mask": [AddControlInputMask], + "control_input_upscale": [AddControlInputUpscale], + "control_input_identity": [AddControlInputIdentity], +} + + +BaseVideoConditionerWithCtrlConfig: LazyDict = L(VideoConditionerWithCtrl)( + text=TextConfig(), +) + +VideoConditionerFpsSizePaddingWithCtrlConfig: LazyDict = L(VideoConditionerWithCtrl)( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/config.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/config.py new file mode 100644 index 00000000..916d7777 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/config.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, List + +import attrs +from cosmos1.models.diffusion.config.ctrl.model import CtrlModelConfig + +# from cosmos1.models.diffusion.config.base.model import DefaultModelConfig +from cosmos1.models.diffusion.config.ctrl.registry import register_configs +from cosmos1.models.diffusion.model.model_ctrl import VideoDiffusionModelWithCtrl +from cosmos1.utils import config +from cosmos1.utils.config_helper import import_all_modules_from_package +from cosmos1.utils.lazy_config import PLACEHOLDER, LazyDict +from cosmos1.utils.lazy_config import LazyCall as L + + +# import cosmos1.models.diffusion.config.config as base_config +# from cosmos1.models.diffusion.checkpointers.ema_fsdp_checkpointer import CheckpointConfig + + +@attrs.define(slots=False) +class Config(config.Config): + # default config groups that will be used unless overwritten + # see config groups in registry.py + defaults: List[Any] = attrs.field( + factory=lambda: [ + "_self_", + {"net": None}, + {"net_ctrl": None}, + {"hint_key": "control_input_canny"}, + {"conditioner": "ctrlnet_add_fps_image_size_padding_mask"}, + {"tokenizer": "vae1"}, + {"experiment": None}, + ] + ) + model_obj: LazyDict = L(VideoDiffusionModelWithCtrl)( + config=PLACEHOLDER, + ) + + +def make_config(): + c = Config( + model=CtrlModelConfig(), + ) + # Specifying values through instances of attrs + + # Call this function to register config groups for advanced overriding. + register_configs() + + # experiment config are defined in the experiment folder + # call import_all_modules_from_package to register them + import_all_modules_from_package("cosmos1.models.diffusion.config.inference.ctrl") + return c diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/model.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/model.py new file mode 100644 index 00000000..ef42a3c9 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/model.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import attrs +from cosmos1.models.diffusion.config.base.model import DefaultModelConfig +from cosmos1.utils.lazy_config import LazyDict + + +@attrs.define(slots=False) +class CtrlModelConfig(DefaultModelConfig): + net_ctrl: LazyDict = None + hint_key: str = None + base_load_from: LazyDict = None + finetune_base_model: bool = False + hint_mask: list = [True] + hint_dropout_rate: float = 0.0 + num_control_blocks: int = 5 + random_drop_control_blocks: bool = False + pixel_corruptor: LazyDict = None diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/net_ctrl.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/net_ctrl.py new file mode 100644 index 00000000..4d667434 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/net_ctrl.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from cosmos1.models.diffusion.config.base.net import FADITV2_14B_Config, FADITV2Config +from cosmos1.models.diffusion.networks.general_dit_ctrl_enc import GeneralDITEncoder + + +num_blocks = FADITV2Config["num_blocks"] +FADITV2EncoderConfig = copy.deepcopy(FADITV2Config) +FADITV2EncoderConfig["_target_"] = GeneralDITEncoder +FADITV2EncoderConfig["layer_mask"] = [True if i > num_blocks // 2 else False for i in range(num_blocks)] + +num_blocks = FADITV2_14B_Config["num_blocks"] +FADITV2_14B_EncoderConfig = copy.deepcopy(FADITV2_14B_Config) +FADITV2_14B_EncoderConfig["_target_"] = GeneralDITEncoder +FADITV2_14B_EncoderConfig["layer_mask"] = [True if i > num_blocks // 2 else False for i in range(num_blocks)] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/registry.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/registry.py new file mode 100644 index 00000000..8ce2d393 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/ctrl/registry.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import cosmos1.models.diffusion.config.registry as base_registry +from cosmos1.models.diffusion.config.ctrl.conditioner import ( + CTRL_HINT_KEYS, + BaseVideoConditionerWithCtrlConfig, + VideoConditionerFpsSizePaddingWithCtrlConfig, +) +from cosmos1.models.diffusion.config.ctrl.net_ctrl import FADITV2_14B_EncoderConfig, FADITV2EncoderConfig +from hydra.core.config_store import ConfigStore + + +def register_experiment_ctrlnet(cs): + cs.store(group="net_ctrl", package="model.net_ctrl", name="faditv2_7b", node=FADITV2EncoderConfig) + cs.store(group="net_ctrl", package="model.net_ctrl", name="faditv2_14b", node=FADITV2_14B_EncoderConfig) + + cs.store(group="conditioner", package="model.conditioner", name="ctrlnet", node=BaseVideoConditionerWithCtrlConfig) + cs.store( + group="conditioner", + package="model.conditioner", + name="ctrlnet_add_fps_image_size_padding_mask", + node=VideoConditionerFpsSizePaddingWithCtrlConfig, + ) + for hint_key in CTRL_HINT_KEYS: + cs.store( + group="hint_key", + package="model", + name=hint_key, + node=dict(hint_key=dict(hint_key=hint_key, grayscale=False)), + ) + cs.store( + group="hint_key", + package="model", + name=f"{hint_key}_grayscale", + node=dict(hint_key=dict(hint_key=hint_key, grayscale=True)), + ) + + +def register_configs(): + cs = ConfigStore.instance() + base_registry.register_configs() + register_experiment_ctrlnet(cs) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1-diffusion-text2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1-diffusion-text2world.py new file mode 100644 index 00000000..b176d909 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1-diffusion-text2world.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cosmos1.utils.lazy_config import LazyDict +from hydra.core.config_store import ConfigStore + + +Cosmos_1_0_Diffusion_Text2World_7B: LazyDict = LazyDict( + dict( + defaults=[ + {"override /net": "faditv2_7b"}, + {"override /conditioner": "add_fps_image_size_padding_mask"}, + {"override /tokenizer": "cosmos_diffusion_tokenizer_res720_comp8x8x8_t121_ver092624"}, + "_self_", + ], + job=dict( + group="Text2World", + name="Cosmos_1_0_Diffusion_Text2World_7B", + ), + model=dict( + latent_shape=[ + 16, + 16, + 88, + 160, + ], + net=dict( + extra_per_block_abs_pos_emb=True, + rope_h_extrapolation_ratio=1.0, + rope_w_extrapolation_ratio=1.0, + rope_t_extrapolation_ratio=2.0, + extra_per_block_abs_pos_emb_type="learnable", + ), + ), + ) +) + + +Cosmos_1_0_Diffusion_Text2World_14B: LazyDict = LazyDict( + dict( + defaults=[ + {"override /net": "faditv2_14b"}, + {"override /conditioner": "add_fps_image_size_padding_mask"}, + {"override /tokenizer": "cosmos_diffusion_tokenizer_res720_comp8x8x8_t121_ver092624"}, + "_self_", + ], + job=dict( + group="Text2World", + name="Cosmos_1_0_Diffusion_Text2World_14B", + ), + model=dict( + latent_shape=[ + 16, + 16, + 88, + 160, + ], + net=dict( + extra_per_block_abs_pos_emb=True, + rope_h_extrapolation_ratio=2.0, + rope_t_extrapolation_ratio=2.0, + rope_w_extrapolation_ratio=2.0, + extra_h_extrapolation_ratio=2.0, + extra_t_extrapolation_ratio=2.0, + extra_w_extrapolation_ratio=2.0, + extra_per_block_abs_pos_emb_type="learnable", + ), + ), + ) +) + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_1_0_Diffusion_Text2World_7B["job"]["name"], + node=Cosmos_1_0_Diffusion_Text2World_7B, +) + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_1_0_Diffusion_Text2World_14B["job"]["name"], + node=Cosmos_1_0_Diffusion_Text2World_14B, +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1-diffusion-video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1-diffusion-video2world.py new file mode 100644 index 00000000..de28ea39 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1-diffusion-video2world.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cosmos1.models.diffusion.networks.general_dit_video_conditioned import VideoExtendGeneralDIT +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict +from hydra.core.config_store import ConfigStore + + +Cosmos_1_0_Diffusion_Video2World_7B: LazyDict = LazyDict( + dict( + defaults=[ + {"override /net": "faditv2_7b"}, + {"override /conditioner": "video_cond"}, + {"override /tokenizer": "cosmos_diffusion_tokenizer_res720_comp8x8x8_t121_ver092624"}, + "_self_", + ], + model=dict( + latent_shape=[ + 16, + 16, + 88, + 160, + ], + conditioner=dict(video_cond_bool=dict()), + net=L(VideoExtendGeneralDIT)( + extra_per_block_abs_pos_emb=True, + rope_h_extrapolation_ratio=1.0, + rope_w_extrapolation_ratio=1.0, + rope_t_extrapolation_ratio=2.0, + extra_per_block_abs_pos_emb_type="learnable", + ), + ), + job=dict(group="Video2World", name="Cosmos_1_0_Diffusion_Video2World_7B"), + ) +) + + +Cosmos_1_0_Diffusion_Video2World_14B: LazyDict = LazyDict( + dict( + defaults=[ + {"override /net": "faditv2_14b"}, + {"override /conditioner": "video_cond"}, + {"override /tokenizer": "cosmos_diffusion_tokenizer_res720_comp8x8x8_t121_ver092624"}, + "_self_", + ], + model=dict( + latent_shape=[ + 16, + 16, + 88, + 160, + ], + conditioner=dict(video_cond_bool=dict()), + net=L(VideoExtendGeneralDIT)( + extra_per_block_abs_pos_emb=True, + rope_h_extrapolation_ratio=2.0, + rope_t_extrapolation_ratio=2.0, + rope_w_extrapolation_ratio=2.0, + extra_h_extrapolation_ratio=2.0, + extra_t_extrapolation_ratio=2.0, + extra_w_extrapolation_ratio=2.0, + extra_per_block_abs_pos_emb_type="learnable", + ), + ), + job=dict(group="Video2World", name="Cosmos_1_0_Diffusion_Video2World_14B"), + ) +) + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_1_0_Diffusion_Video2World_7B["job"]["name"], + node=Cosmos_1_0_Diffusion_Video2World_7B, +) + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_1_0_Diffusion_Video2World_14B["job"]["name"], + node=Cosmos_1_0_Diffusion_Video2World_14B, +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1_1-diffusion-multi-camera.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1_1-diffusion-multi-camera.py new file mode 100644 index 00000000..d129c41f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/inference/cosmos-1_1-diffusion-multi-camera.py @@ -0,0 +1,93 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from cosmos1.models.diffusion.networks.general_dit_multi_camera import MultiCameraVideoExtendGeneralDIT +from cosmos1.utils.lazy_config import LazyCall as L +from cosmos1.utils.lazy_config import LazyDict +from hydra.core.config_store import ConfigStore + + +Cosmos_1_1_Diffusion_Multi_Camera_Text2World_7B: LazyDict = LazyDict( + dict( + defaults=[ + "/experiment/Cosmos_1_0_Diffusion_Text2World_7B", + {"override /net": "faditv2_multicam_7b"}, + {"override /conditioner": "add_fps_image_size_padding_mask_frame_repeat"}, + "_self_", + ], + job=dict( + group="Text2World", + name="Cosmos_1_1_Diffusion_Multi_Camera_Text2World_7B", + ), + model=dict( + latent_shape=[ + 16, + 16, + 88, + 160, + ], + tokenizer=dict( + video_vae=dict( + pixel_chunk_duration=57, + ) + ), + ), + ) +) + +Cosmos_1_1_Diffusion_Multi_Camera_Video2World_7B: LazyDict = LazyDict( + dict( + defaults=[ + "/experiment/Cosmos_1_1_Diffusion_Multi_Camera_Text2World_7B", + {"override /conditioner": "video_cond_frame_repeat"}, + "_self_", + ], + job=dict( + group="Text2World", + name="Cosmos_1_1_Diffusion_Multi_Camera_Video2World_7B", + ), + model=dict( + latent_shape=[ + 16, + 16, + 88, + 160, + ], + net=L(MultiCameraVideoExtendGeneralDIT)( + n_cameras=6, + camera_condition_dim=6, + add_repeat_frame_embedding=True, + ), + conditioner=dict(video_cond_bool=dict()), + ), + ) +) + + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_1_1_Diffusion_Multi_Camera_Text2World_7B["job"]["name"], + node=Cosmos_1_1_Diffusion_Multi_Camera_Text2World_7B, +) + + +cs = ConfigStore.instance() +cs.store( + group="experiment", + package="_global_", + name=Cosmos_1_1_Diffusion_Multi_Camera_Video2World_7B["job"]["name"], + node=Cosmos_1_1_Diffusion_Multi_Camera_Video2World_7B, +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/registry.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/registry.py new file mode 100644 index 00000000..f65318b6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/config/registry.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from cosmos1.models.diffusion.config.base.conditioner import ( + BaseVideoConditionerConfig, + VideoConditionerFpsSizePaddingConfig, + VideoConditionerFpsSizePaddingFrameRepeatConfig, + VideoExtendConditionerConfig, + VideoExtendConditionerFrameRepeatConfig, +) +from cosmos1.models.diffusion.config.base.net import FADITV2_14B_Config, FADITV2_MultiCam_Config, FADITV2Config +from cosmos1.models.diffusion.config.base.tokenizer import get_cosmos_diffusion_tokenizer_comp8x8x8 +from hydra.core.config_store import ConfigStore + + +def register_net(cs): + cs.store( + group="net", + package="model.net", + name="faditv2_7b", + node=FADITV2Config, + ) + cs.store( + group="net", + package="model.net", + name="faditv2_14b", + node=FADITV2_14B_Config, + ) + cs.store( + group="net", + package="model.net", + name="faditv2_multicam_7b", + node=FADITV2_MultiCam_Config, + ) + + +def register_conditioner(cs): + cs.store( + group="conditioner", + package="model.conditioner", + name="basic", + node=BaseVideoConditionerConfig, + ) + cs.store( + group="conditioner", + package="model.conditioner", + name="add_fps_image_size_padding_mask", + node=VideoConditionerFpsSizePaddingConfig, + ) + cs.store( + group="conditioner", + package="model.conditioner", + name="video_cond", + node=VideoExtendConditionerConfig, + ) + cs.store( + group="conditioner", + package="model.conditioner", + name="add_fps_image_size_padding_mask_frame_repeat", + node=VideoConditionerFpsSizePaddingFrameRepeatConfig, + ) + cs.store( + group="conditioner", + package="model.conditioner", + name="video_cond_frame_repeat", + node=VideoExtendConditionerFrameRepeatConfig, + ) + + +def register_tokenizer(cs): + cs.store( + group="tokenizer", + package="model.tokenizer", + name="cosmos_diffusion_tokenizer_res720_comp8x8x8_t121_ver092624", + node=get_cosmos_diffusion_tokenizer_comp8x8x8(resolution="720", chunk_duration=121), + ) + + +def register_configs(): + cs = ConfigStore.instance() + + register_net(cs) + register_conditioner(cs) + register_tokenizer(cs) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/control_input.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/control_input.py new file mode 100644 index 00000000..1fe07765 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/control_input.py @@ -0,0 +1,1622 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import random +from functools import partial +from typing import Any, Optional + +import cv2 +import matplotlib.colors as mcolors +import numpy as np +import pycocotools +import torch +import torchvision.transforms.functional as transforms_F +from cosmos1.models.diffusion.config.ctrl.blurs import ( + AnisotropicDiffusionConfig, + BilateralFilterConfig, + BlurAugmentorConfig, + GaussianBlurConfig, + GuidedFilterConfig, + LaplacianOfGaussianConfig, + MedianBlurConfig, +) +from cosmos1.models.diffusion.datasets.augmentors.guided_filter import FastGuidedFilter +from cosmos1.models.diffusion.datasets.augmentors.human_keypoint_utils import coco_wholebody_133_skeleton +from cosmos1.utils import log + + +IMAGE_RES_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": { # the image format does not support 1080, but here we match it with video resolution + "1,1": (1024, 1024), + "4,3": (1440, 1056), + "3,4": (1056, 1440), + "16,9": (1920, 1056), + "9,16": (1056, 1920), + }, + "1024": {"1,1": (1024, 1024), "4,3": (1280, 1024), "3,4": (1024, 1280), "16,9": (1280, 768), "9,16": (768, 1280)}, + # 720; mainly for make sure it matches video resolution conventions + "720": {"1,1": (960, 960), "4,3": (960, 704), "3,4": (704, 960), "16,9": (1280, 704), "9,16": (704, 1280)}, + "512": {"1,1": (512, 512), "4,3": (640, 512), "3,4": (512, 640), "16,9": (640, 384), "9,16": (384, 640)}, + "256": { + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (320, 192), + "9,16": (192, 320), + }, +} + + +VIDEO_RES_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": { # 1080p doesn't have 1:1 + "1,1": (1024, 1024), + "4,3": (1440, 1056), + "3,4": (1056, 1440), + "16,9": (1920, 1056), + "9,16": (1056, 1920), + }, + # 1024; the video format does not support it, but here we match it with image resolution + "1024": {"1,1": (1024, 1024), "4,3": (1280, 1024), "3,4": (1024, 1280), "16,9": (1280, 768), "9,16": (768, 1280)}, + "720": {"1,1": (960, 960), "4,3": (960, 704), "3,4": (704, 960), "16,9": (1280, 704), "9,16": (704, 1280)}, + "512": {"1,1": (512, 512), "4,3": (640, 512), "3,4": (512, 640), "16,9": (640, 384), "9,16": (384, 640)}, + "480": {"1,1": (480, 480), "4,3": (640, 480), "3,4": (480, 640), "16,9": (768, 432), "9,16": (432, 768)}, + "256": { + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (320, 192), + "9,16": (192, 320), + }, +} + + +class Augmentor: + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + r"""Base augmentor class + + Args: + input_keys (list): List of input keys + output_keys (list): List of output keys + args (dict): Arguments associated with the augmentation + """ + self.input_keys = input_keys + self.output_keys = output_keys + self.args = args + + def __call__(self, *args: Any, **kwds: Any) -> Any: + raise ValueError("Augmentor not implemented") + + +def resize_frames(frames, is_image, data_dict): + # Resize the frames to target size before computing control signals to save compute. + need_reshape = len(frames.shape) < 4 + if need_reshape: # HWC -> CTHW + frames = frames.transpose((2, 0, 1))[:, None] + H, W = frames.shape[2], frames.shape[3] + + if "__url__" in data_dict and "aspect_ratio" in data_dict["__url__"].meta.opts: + aspect_ratio = data_dict["__url__"].meta.opts["aspect_ratio"] + elif "aspect_ratio" in data_dict: # Non-webdataset format + aspect_ratio = data_dict["aspect_ratio"] + else: + aspect_ratio = "16,9" + RES_SIZE_INFO = IMAGE_RES_SIZE_INFO if is_image else VIDEO_RES_SIZE_INFO + new_W, new_H = RES_SIZE_INFO["720"][aspect_ratio] + scaling_ratio = min((new_W / W), (new_H / H)) + if scaling_ratio < 1: + W, H = int(scaling_ratio * W + 0.5), int(scaling_ratio * H + 0.5) + frames = [ + cv2.resize(_image_np, (W, H), interpolation=cv2.INTER_AREA) for _image_np in frames.transpose((1, 2, 3, 0)) + ] + frames = np.stack(frames).transpose((3, 0, 1, 2)) + if need_reshape: # CTHW -> HWC + frames = frames[:, 0].transpose((1, 2, 0)) + return frames + + +# frames CTHW +def apply_gaussian_blur(frames: np.ndarray, ksize: int = 5, sigmaX: float = 1.0) -> np.ndarray: + if ksize % 2 == 0: + ksize += 1 # ksize must be odd + blurred_image = [ + cv2.GaussianBlur(_image_np, (ksize, ksize), sigmaX=sigmaX) for _image_np in frames.transpose((1, 2, 3, 0)) + ] + blurred_image = np.stack(blurred_image).transpose((3, 0, 1, 2)) + return blurred_image + + +class GaussianBlur: + def __init__(self, config: GaussianBlurConfig) -> None: + self.use_random = config.use_random + self.config = config + + def __call__(self, frames: np.ndarray) -> np.ndarray: + if self.use_random: + ksize = np.random.randint(self.config.ksize_min, self.config.ksize_max + 1) + sigmaX = np.random.uniform(self.config.sigmaX_min, self.config.sigmaX_max) + else: + ksize = self.config.ksize + sigmaX = self.config.sigmaX + return apply_gaussian_blur(frames, ksize, sigmaX) + + +def apply_guided_filter(frames: np.ndarray, radius: int, eps: float, scale: float) -> np.ndarray: + blurred_image = [ + FastGuidedFilter(_image_np, radius, eps, scale).filter(_image_np) + for _image_np in frames.transpose((1, 2, 3, 0)) + ] + blurred_image = np.stack(blurred_image).transpose((3, 0, 1, 2)) + return blurred_image + + +class GuidedFilter: + def __init__(self, config: GuidedFilterConfig) -> None: + self.use_random = config.use_random + self.config = config + + def __call__(self, frames: np.ndarray) -> np.ndarray: + if self.use_random: + radius = np.random.randint(self.config.radius_min, self.config.radius_max + 1) + eps = np.random.uniform(self.config.eps_min, self.config.eps_max) + scale = np.random.randint(self.config.scale_min, self.config.scale_max + 1) + else: + radius = self.config.radius + eps = self.config.eps + scale = self.config.scale + return apply_guided_filter(frames, radius, eps, scale) + + +def apply_bilateral_filter( + frames: np.ndarray, + d: int = 9, + sigma_color: float = 75, + sigma_space: float = 75, + iter: int = 1, +) -> np.ndarray: + blurred_image = [] + for _image_np in frames.transpose((1, 2, 3, 0)): + for _ in range(iter): + _image_np = cv2.bilateralFilter(_image_np, d, sigma_color, sigma_space) + blurred_image += [_image_np] + + blurred_image = np.stack(blurred_image).transpose((3, 0, 1, 2)) + return blurred_image + + +class BilateralFilter: + def __init__(self, config: BilateralFilterConfig) -> None: + self.use_random = config.use_random + self.config = config + + def __call__(self, frames: np.ndarray) -> np.ndarray: + config = self.config + if self.use_random: + d = np.random.randint(config.d_min, config.d_max) + sigma_color = np.random.randint(config.sigma_color_min, config.sigma_color_max) + sigma_space = np.random.randint(config.sigma_space_min, config.sigma_space_max) + iter = np.random.randint(config.iter_min, config.iter_max) + else: + d = config.d + sigma_color = config.sigma_color + sigma_space = config.sigma_space + iter = config.iter + return apply_bilateral_filter(frames, d, sigma_color, sigma_space, iter) + + +def apply_median_blur(frames: np.ndarray, ksize=5) -> np.ndarray: + if ksize % 2 == 0: + ksize += 1 # ksize must be odd + blurred_image = [cv2.medianBlur(_image_np, ksize) for _image_np in frames.transpose((1, 2, 3, 0))] + blurred_image = np.stack(blurred_image).transpose((3, 0, 1, 2)) + return blurred_image + + +class MedianBlur: + def __init__(self, config: MedianBlurConfig) -> None: + self.use_random = config.use_random + self.config = config + + def __call__(self, frames: np.ndarray) -> np.ndarray: + if self.use_random: + ksize = np.random.randint(self.config.ksize_min, self.config.ksize_max + 1) + else: + ksize = self.config.ksize + return apply_median_blur(frames, ksize) + + +def apply_laplacian_of_gaussian( + frames: np.ndarray, ksize: int = 5, sigma: float = 1.4, binarize: bool = False, threshold: float = 0.0 +) -> np.ndarray: + """ + Apply Laplacian of Gaussian edge detection to a set of frames. + + Args: + frames (np.ndarray): Input frames with shape (C, T, H, W) + ksize (int): Size of the Gaussian kernel. Must be odd and positive. + sigma (float): Standard deviation of the Gaussian distribution. + binarize (bool): Whether to binarize the output edge map. + threshold (float): Threshold for binarization (if binarize is True). + + Returns: + np.ndarray: Edge-detected frames with shape (C, T, H, W). + """ + # Ensure ksize is odd + if ksize % 2 == 0: + ksize += 1 # ksize must be odd + + edge_frames = [] + for frame in frames.transpose((1, 2, 3, 0)): # (T, H, W, C) + # Convert to grayscale if the image is in color + if frame.shape[-1] == 3: + gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) + else: + gray = frame.squeeze() + + blurred = cv2.GaussianBlur(gray, (ksize, ksize), sigma) + laplacian = cv2.Laplacian(blurred, cv2.CV_64F) + normalized = cv2.normalize(laplacian, None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U) + + if binarize: + _, edge_map = cv2.threshold(normalized, threshold, 255, cv2.THRESH_BINARY) + else: + edge_map = normalized + + # Expand dimensions to match input shape + edge_map = np.repeat(edge_map[..., np.newaxis], frames.shape[0], axis=-1) + edge_frames.append(edge_map) + return np.stack(edge_frames).transpose((3, 0, 1, 2)) # (C, T, H, W) + + +class LaplacianOfGaussian: + """ + Applies Laplacian of Gaussian edge detection to images or video frames. + """ + + def __init__(self, config: LaplacianOfGaussianConfig) -> None: + self.use_random = config.use_random + self.config = config + + def __call__(self, frames: np.ndarray) -> np.ndarray: + """Apply LoG to input frames.""" + if self.use_random: + ksize = np.random.randint(self.config.ksize_min, self.config.ksize_max + 1) + sigma = np.random.uniform(self.config.sigma_min, self.config.sigma_max) + binarize = np.random.choice([True, False]) if not self.config.binarize else self.config.binarize + threshold = np.random.uniform(self.config.threshold_min, self.config.threshold_max) if binarize else 0 + else: + ksize = self.config.ksize + sigma = self.config.sigma + binarize = self.config.binarize + threshold = self.config.threshold + return apply_laplacian_of_gaussian(frames, ksize, sigma, binarize, threshold) + + +def apply_anisotropic_diffusion(frames: np.ndarray, alpha: float, K: float, niters: int) -> np.ndarray: + """ + Apply Anisotropic Diffusion to a set of frames. + + Args: + frames (np.ndarray): Input frames with shape (C, T, H, W) + alpha (float): The amount of time to step forward on each iteration (between 0 and 1) + K (float): Sensitivity to edges + niters (int): Number of iterations + + Returns: + np.ndarray: Anisotropic-diffused frames with shape (C, T, H, W). + """ + blurred_image = [ + cv2.ximgproc.anisotropicDiffusion(_image_np, alpha, K, niters) for _image_np in frames.transpose((1, 2, 3, 0)) + ] + blurred_image = np.stack(blurred_image).transpose((3, 0, 1, 2)) + + return blurred_image + + +class AnisotropicDiffusion: + """ + Applies Anisotropic Diffusion to images or video frames. + """ + + def __init__(self, config: AnisotropicDiffusionConfig) -> None: + self.use_random = config.use_random + self.config = config + + def __call__(self, frames: np.ndarray) -> np.ndarray: + if self.use_random: + alpha = np.random.uniform(self.config.alpha_min, self.config.alpha_max) + K = np.random.uniform(self.config.K_min, self.config.K_max) + niters = np.random.randint(self.config.niters_min, self.config.niters_max + 1) + else: + alpha = self.config.alpha + K = self.config.K + niters = self.config.niters + return apply_anisotropic_diffusion(frames, alpha, K, niters) + + +class Blur: + def __init__(self, config: BlurAugmentorConfig, output_key: str = "") -> None: + self.output_key = output_key if output_key else None + + probabilities = [combo.probability for combo in config.blur_combinations] + total_prob = sum(probabilities) + assert abs(total_prob - 1.0) < 1e-6, f"Probabilities must sum to 1.0, got {total_prob}" + + self.blur_combinations = config.blur_combinations + self.downscale_factor = config.downscale_factor + self.probabilities = probabilities + self._set_blur_instances() + + def _set_blur_instances(self): + if not self.blur_combinations: + return + self.blur_combinations_instances = [] + + for blur_combination in self.blur_combinations: + blur_mapping = { + "gaussian": (GaussianBlur, blur_combination.gaussian_blur), + "guided": (GuidedFilter, blur_combination.guided_filter), + "bilateral": (BilateralFilter, blur_combination.bilateral_filter), + "median": (MedianBlur, blur_combination.median_blur), + "log": (LaplacianOfGaussian, blur_combination.log), + "anisotropic": (AnisotropicDiffusion, blur_combination.anisotropic_diffusion), + } + + cur_instances = [] + for blur_type in blur_combination.blur_types: + assert blur_type in blur_mapping, f"Unknown {blur_type}. Needs to correct blur_type or blur_mapping." + + blur_class, blur_config = blur_mapping[blur_type] + cur_instances.append(blur_class(blur_config)) + + self.blur_combinations_instances.append(cur_instances) + + assert len(self.blur_combinations_instances) == len(self.blur_combinations), ( + "Number of blur_combinations_instances needs to match number of blur_combinations." + ) + + def __call__(self, frames: np.ndarray) -> np.ndarray: + blur_instances = random.choices(self.blur_combinations_instances, weights=self.probabilities, k=1)[0] + + H, W = frames.shape[2], frames.shape[3] + downscale_factor = random.choice(self.downscale_factor) + if downscale_factor > 1: + frames = [ + cv2.resize(_image_np, (W // downscale_factor, H // downscale_factor), interpolation=cv2.INTER_AREA) + for _image_np in frames.transpose((1, 2, 3, 0)) + ] + frames = np.stack(frames).transpose((3, 0, 1, 2)) + + for ins in blur_instances: + frames = ins(frames) + + if downscale_factor > 1: + frames = [ + cv2.resize(_image_np, (W, H), interpolation=cv2.INTER_LINEAR) + for _image_np in frames.transpose((1, 2, 3, 0)) + ] + frames = np.stack(frames).transpose((3, 0, 1, 2)) + return frames + + +class AddControlInputBlurDownUp(Augmentor): + """ + Main class for adding blurred input to the data dictionary. + self.output_keys[0] indicates the types of blur added to the input. + For example, control_input_gaussian_guided indicates that both Gaussian and Guided filters are applied + """ + + def __init__( + self, + input_keys: list, # [key_load, key_img] + output_keys: Optional[list] = [ + "control_input_gaussian_guided_bilateral_median_log" + ], # eg ["control_input_gaussian_guided"] + args: Optional[dict] = None, # not used + use_random: bool = True, # whether to use random parameters + blur_config: BlurAugmentorConfig = BlurAugmentorConfig(), + downup_preset: str | int = "medium", # preset strength for downup factor + min_downup_factor: int = 4, # minimum downup factor + max_downup_factor: int = 16, # maximum downup factor + downsize_before_blur: bool = False, # whether to downsize before applying blur and then upsize or downup after blur + ) -> None: + super().__init__(input_keys, output_keys, args) + self.use_random = use_random + downup_preset_values = { + "none": 1, + "very_low": min_downup_factor, + "low": min_downup_factor, + "medium": (min_downup_factor + max_downup_factor) // 2, + "high": max_downup_factor, + "very_high": max_downup_factor, + } + + self.blur = Blur(config=blur_config, output_key=self.output_keys[0]) + + self.downup_preset = downup_preset if isinstance(downup_preset, int) else downup_preset_values[downup_preset] + self.downsize_before_blur = downsize_before_blur + self.min_downup_factor = min_downup_factor + self.max_downup_factor = max_downup_factor + + def _load_frame(self, data_dict: dict) -> tuple[np.ndarray, bool]: + key_img = self.input_keys[1] + frames = data_dict[key_img] + frames = np.array(frames) + is_image = False + if len(frames.shape) < 4: + frames = frames.transpose((2, 0, 1))[:, None] + is_image = True + return frames, is_image + + def __call__(self, data_dict: dict) -> dict: + key_out = self.output_keys[0] + frames, is_image = self._load_frame(data_dict) + # H, W = frames.shape[2], frames.shape[3] + + # Resize the frames to target size before blurring. + frames = resize_frames(frames, is_image, data_dict) + H, W = frames.shape[2], frames.shape[3] + # if is_image: + # data_dict[key_img] = torch.from_numpy(frames)[:, 0] + # else: + # data_dict[key_img] = torch.from_numpy(frames) + + if self.use_random: + scale_factor = random.randint(self.min_downup_factor, self.max_downup_factor + 1) + else: + scale_factor = self.downup_preset + if self.downsize_before_blur: + frames = [ + cv2.resize(_image_np, (W // scale_factor, H // scale_factor), interpolation=cv2.INTER_AREA) + for _image_np in frames.transpose((1, 2, 3, 0)) + ] + frames = np.stack(frames).transpose((3, 0, 1, 2)) + frames = self.blur(frames) + if self.downsize_before_blur: + frames = [ + cv2.resize(_image_np, (W, H), interpolation=cv2.INTER_LINEAR) + for _image_np in frames.transpose((1, 2, 3, 0)) + ] + frames = np.stack(frames).transpose((3, 0, 1, 2)) + if is_image: + frames = frames[:, 0] + # turn into tensor + controlnet_img = torch.from_numpy(frames) + if not self.downsize_before_blur: + # Resize image + controlnet_img = transforms_F.resize( + controlnet_img, + size=(int(H / scale_factor), int(W / scale_factor)), + interpolation=transforms_F.InterpolationMode.BICUBIC, + antialias=True, + ) + controlnet_img = transforms_F.resize( + controlnet_img, + size=(H, W), + interpolation=transforms_F.InterpolationMode.BICUBIC, + antialias=True, + ) + data_dict[key_out] = controlnet_img + return data_dict + + +class AddControlInputCanny(Augmentor): + """ + Add control input to the data dictionary. control input are expanded to 3-channels + steps to add new items: modify this file, configs/conditioner.py, conditioner.py + """ + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_canny"], + args: Optional[dict] = None, + use_random: bool = True, + preset_strength="medium", + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + self.use_random = use_random + self.preset_strength = preset_strength + + def __call__(self, data_dict: dict) -> dict: + key_img = self.input_keys[1] + key_out = self.output_keys[0] + frames = data_dict[key_img] + # Get lower and upper threshold for canny edge detection. + if self.use_random: # always on for training, always off for inference + t_lower = np.random.randint(20, 100) # Get a random lower thre within [0, 255] + t_diff = np.random.randint(50, 150) # Get a random diff between lower and upper + t_upper = min(255, t_lower + t_diff) # The upper thre is lower added by the diff + else: + if self.preset_strength == "none" or self.preset_strength == "very_low": + t_lower, t_upper = 20, 50 + elif self.preset_strength == "low": + t_lower, t_upper = 50, 100 + elif self.preset_strength == "medium": + t_lower, t_upper = 100, 200 + elif self.preset_strength == "high": + t_lower, t_upper = 200, 300 + elif self.preset_strength == "very_high": + t_lower, t_upper = 300, 400 + else: + raise ValueError(f"Preset {self.preset_strength} not recognized.") + frames = np.array(frames) + is_image = len(frames.shape) < 4 + + # Resize the frames to target size before computing canny edges. + frames = resize_frames(frames, is_image, data_dict) + + # Compute the canny edge map by the two thresholds. + if is_image: + edge_maps = cv2.Canny(frames, t_lower, t_upper)[None, None] + else: + edge_maps = [cv2.Canny(img, t_lower, t_upper) for img in frames.transpose((1, 2, 3, 0))] + edge_maps = np.stack(edge_maps)[None] + edge_maps = torch.from_numpy(edge_maps).expand(3, -1, -1, -1) + if is_image: + edge_maps = edge_maps[:, 0] + data_dict[key_out] = edge_maps + return data_dict + + +class AddControlInputIdentity(Augmentor): + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_identity"], + args: Optional[dict] = None, + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + key_img = self.input_keys[1] + key_out = self.output_keys[0] + frames = np.array(data_dict[key_img]) # CTHW for video, HWC for image + is_image = len(frames.shape) < 4 + if is_image: + frames = frames.transpose((2, 0, 1)) + data_dict[key_out] = torch.from_numpy(frames).clone() # CTHW for video, CHW for image + return data_dict + + +class AddControlInput(Augmentor): + """ + For backward compatibility. The previously trained models use legacy_process + """ + + def __init__( + self, + input_keys: list, + output_keys=["control_input_gaussian_guided_bilateral_median_log_anisotropic"], + args=None, + blur_config: BlurAugmentorConfig = BlurAugmentorConfig(), + use_random=True, + preset_strength="medium", + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + + self.process = AddControlInputBlurDownUp( + input_keys, + output_keys, + args, + blur_config=blur_config, + downup_preset=preset_strength, # preset strength for downup factor + use_random=use_random, + ) + + def __call__(self, data_dict: dict) -> dict: + return self.process(data_dict) + + +class AddControlInputComb(Augmentor): + """ + Add control input to the data dictionary. control input are expanded to 3-channels + steps to add new items: modify this file, configs/conditioner.py, conditioner.py + """ + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = None, + blur_config: BlurAugmentorConfig = None, + args: Optional[dict] = None, + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + assert "comb" in args + self.comb = {} + for class_name in args["comb"]: + if class_name in [AddControlInput, AddControlInputBlurDownUp]: + aug = class_name(input_keys=input_keys, args=args, blur_config=blur_config, **kwargs) + else: + aug = class_name(input_keys=input_keys, args=args, **kwargs) + + key = aug.output_keys[0] + self.comb[key] = aug + + def __call__(self, data_dict: dict) -> dict: + all_comb = [] + for k, v in self.comb.items(): + data_dict = v(data_dict) + all_comb.append(data_dict.pop(k)) + if all_comb[-1].dim() == 4: + all_comb[-1] = all_comb[-1].squeeze(1) + all_comb = torch.cat(all_comb, dim=0) + data_dict[self.output_keys[0]] = all_comb + return data_dict + + +def get_augmentor_for_eval( + input_key: str, + output_key: str, + blur_config: BlurAugmentorConfig = BlurAugmentorConfig(), + preset_strength: str = "medium", + blur_type: str = "gaussian,guided,bilateral,median,log,anisotropic", # do we still need this value? +) -> AddControlInputComb: + comb = [] + if "canny" in output_key: + comb.append(partial(AddControlInputCanny, output_keys=["control_input_canny"])) + if "upscale" in output_key: + comb.append(partial(AddControlInputUpscale, output_keys=["control_input_upscale"])) + if "identity" in output_key: + comb.append(partial(AddControlInputIdentity, output_keys=["control_input_identity"])) + if "depth" in output_key: + comb.append(partial(AddControlInputDepth, output_keys=["control_input_depth"])) + if "segmentation" in output_key: + comb.append(partial(AddControlInputSeg, output_keys=["control_input_segmentation"])) + if "blur" in output_key: + comb.append(AddControlInput) + process = AddControlInputComb( + input_keys=["", input_key], + output_keys=[output_key], + args={"comb": comb}, + blur_config=blur_config, + use_random=False, + preset_strength=preset_strength, + ) + return process + + +class AddControlInputDepth(Augmentor): + """ + Add control input to the data dictionary. control input are expanded to 3-channels + steps to add new items: modify this file, configs/conditioner.py, conditioner.py + """ + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_depth"], + args: Optional[dict] = None, + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + if "control_input_depth" in data_dict: + # already processed + return data_dict + if "video" not in data_dict: + image = np.array(data_dict[self.input_keys[1]]) + h, w, _ = image.shape + data_dict[self.output_keys[0]] = torch.from_numpy(np.zeros((3, h, w)).astype(np.uint8)) + return data_dict + + assert data_dict["chunk_index"] == data_dict["depth"]["chunk_index"] + key_out = self.output_keys[0] + depth = data_dict["depth"]["video"] + # depth = resize_frames(depth, False, data_dict) + data_dict[key_out] = depth + return data_dict + + +# Array of 23 highly distinguishable colors in RGB format +PREDEFINED_COLORS_SEGMENTATION = np.array( + [ + [255, 0, 0], # Red + [0, 255, 0], # Green + [0, 0, 255], # Blue + [255, 255, 0], # Yellow + [0, 255, 255], # Cyan + [255, 0, 255], # Magenta + [255, 140, 0], # Dark Orange + [255, 105, 180], # Hot Pink + [0, 0, 139], # Dark Blue + [0, 128, 128], # Teal + [75, 0, 130], # Indigo + [128, 0, 128], # Purple + [255, 69, 0], # Red-Orange + [34, 139, 34], # Forest Green + [128, 128, 0], # Olive + [70, 130, 180], # Steel Blue + [255, 215, 0], # Gold + [255, 222, 173], # Navajo White + [144, 238, 144], # Light Green + [255, 99, 71], # Tomato + [221, 160, 221], # Plum + [0, 255, 127], # Spring Green + [255, 255, 255], # White + ] +) + + +def generate_distinct_colors(): + """ + Generate `n` visually distinguishable and randomized colors. + + Returns: + np.ndarray, (3) + """ + # Randomize hue, saturation, and lightness within a range + hue = random.uniform(0, 1) # Full spectrum of hues + saturation = random.uniform(0.1, 1) # Vibrant colors + lightness = random.uniform(0.2, 1.0) # Avoid too dark + + r, g, b = mcolors.hsv_to_rgb((hue, saturation, lightness)) + return (np.array([r, g, b]) * 255).astype(np.uint8) + + +def segmentation_color_mask(segmentation_mask: np.ndarray, use_fixed_color_list: bool = False) -> np.ndarray: + """ + Convert segmentation mask to color mask + Args: + segmentation_mask: np.ndarray, shape (num_masks, T, H, W) + Returns: + np.ndarray, shape (3, T, H, W), with each mask converted to a color mask, value [0,255] + """ + + num_masks, T, H, W = segmentation_mask.shape + segmentation_mask_sorted = [segmentation_mask[i] for i in range(num_masks)] + # Sort the segmentation mask by the number of non-zero pixels, from most to least + segmentation_mask_sorted = sorted(segmentation_mask_sorted, key=lambda x: np.count_nonzero(x), reverse=True) + + output = np.zeros((3, T, H, W), dtype=np.uint8) + if use_fixed_color_list: + predefined_colors_permuted = PREDEFINED_COLORS_SEGMENTATION[ + np.random.permutation(len(PREDEFINED_COLORS_SEGMENTATION)) + ] + else: + predefined_colors_permuted = [generate_distinct_colors() for _ in range(num_masks)] + # index the segmentation mask from last channel to first channel, i start from num_masks-1 to 0 + for i in range(num_masks): + mask = segmentation_mask_sorted[i] + color = predefined_colors_permuted[i % len(predefined_colors_permuted)] + + # Create boolean mask and use it for assignment + bool_mask = mask > 0 + for c in range(3): + output[c][bool_mask] = color[c] + + return output + + +def decode_partial_rle_width1(rle_obj, start_row, end_row): + """ + Decode a partial RLE encoded mask with width = 1. In SAM2 output, the video mask (num_frame, height, width) are reshaped to (total_size, 1). + Sometimes the video mask could be large, e.g. 1001x1080x1092 shape and it takes >1GB memory if using pycocotools, resulting in segmentation faults when training with multiple GPUs and data workers. + This function is used to decode the mask for a subset of frames to reduce memory usage. + + Args: + rle_obj (dict): RLE object containing: + - 'size': A list [height, width=1] indicating the dimensions of the mask. + - 'counts': A bytes or string object containing the RLE encoded data. + start_row (int): The starting row (inclusive). It's computed from frame_start * height * width. + end_row (int): The ending row (exclusive). It's computed from frame_end * height * width. + + Returns: + numpy.ndarray: Decoded binary mask for the specified rows as a 1D numpy array. + """ + height, width = rle_obj["size"] + + # Validate row range + if width != 1: + raise ValueError("This function is optimized for width=1.") + if start_row < 0 or end_row > height or start_row >= end_row: + raise ValueError("Invalid row range specified.") + + # Decode the RLE counts + counts = rle_obj["counts"] + if isinstance(counts, str): + counts = np.frombuffer(counts.encode("ascii"), dtype=np.uint8) + elif isinstance(counts, bytes): + counts = np.frombuffer(counts, dtype=np.uint8) + else: + raise ValueError("Unsupported format for counts. Must be str or bytes.") + + # Interpret counts as a sequence of run lengths + run_lengths = [] + current_val = 0 + i = 0 + while i < len(counts): + x = 0 + k = 0 + more = True + while more: + c = counts[i] - 48 + x |= (c & 0x1F) << (5 * k) + more = (c & 0x20) != 0 + i += 1 + k += 1 + if not more and (c & 0x10): + x |= -1 << (5 * k) + if len(run_lengths) > 2: + x += run_lengths[-2] + + run_lengths.append(x) + current_val += x + if current_val > end_row: + break + # Initialize the partial mask + idx_start = start_row + idx_end = end_row + partial_mask = np.zeros(idx_end - idx_start, dtype=np.uint8) + partial_height = end_row - start_row + idx = 0 # Current global index + for i, run in enumerate(run_lengths): + run_start = idx + run_end = idx + run + if run_end <= idx_start: + # Skip runs entirely before the region + idx = run_end + continue + if run_start >= idx_end: + # Stop decoding once we pass the region + break + + # Calculate overlap with the target region + start = max(run_start, idx_start) + end = min(run_end, idx_end) + if start < end: + partial_start = start - idx_start + partial_end = end - idx_start + partial_mask[partial_start:partial_end] = i % 2 + + idx = run_end + return partial_mask.reshape((partial_height, 1), order="F") + + +class AddControlInputSeg(Augmentor): + """ + Add control input to the data dictionary. control input are expanded to 3-channels + steps to add new items: modify this file, configs/conditioner.py, conditioner.py + """ + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_segmentation"], + thres_mb_python_decode: Optional[int] = 256, # required: <= 512 for 7b + use_fixed_color_list: bool = False, + num_masks_max: int = 100, + random_sample_num_masks: bool = True, + args: Optional[dict] = None, + **kwargs, + ) -> None: + """ + Args: + thres_mb_python_decode: int, threshold of memory usage for python decode, in MB + use_fixed_color_list: bool, if True, use predefined colors for segmentation masks. If False, generate random colors for segmentation masks. + num_masks_max: int, maximum number of masks to sample + random_sample_num_masks: bool, if True, sample number of masks randomly. If False, sample all masks in the data. + + """ + super().__init__(input_keys, output_keys, args) + self.use_fixed_color_list = use_fixed_color_list + self.num_masks_max = num_masks_max + self.thres_mb_python_decode = thres_mb_python_decode + self.random_sample_num_masks = random_sample_num_masks + + def __call__(self, data_dict: dict) -> dict: + if "control_input_segmentation" in data_dict: + # already processed + log.info( + f"control_input_segmentation already processed, shape={data_dict['control_input_segmentation'].shape}, dtype={data_dict['control_input_segmentation'].dtype}, value range: {data_dict['control_input_segmentation'].min()}, {data_dict['control_input_segmentation'].max()}" + ) + return data_dict + if "video" not in data_dict: + image = np.array(data_dict[self.input_keys[1]]) + h, w, _ = image.shape + data_dict[self.output_keys[0]] = torch.from_numpy(np.zeros((3, h, w)).astype(np.uint8)) + return data_dict + frames = data_dict["video"] + _, T, H, W = frames.shape + + all_masks = [] + # sample number of masks + if self.random_sample_num_masks: + num_masks = np.random.randint(0, min(self.num_masks_max + 1, len(data_dict["segmentation"]) + 1)) + else: + num_masks = len(data_dict["segmentation"]) + mask_ids = np.arange(len(data_dict["segmentation"])).tolist() + mask_ids_select = np.random.choice(mask_ids, num_masks, replace=False) + # concat phrases + segmentation_phrase_all = [data_dict["segmentation"][mid]["phrase"] for mid in mask_ids_select] + segmentation_phrase_all = ";".join(segmentation_phrase_all) + data_dict["segmentation_phrase_all"] = segmentation_phrase_all + # obtrain frame indices + frame_start = data_dict["frame_start"] + frame_end = data_dict["frame_end"] + frame_indices = np.arange(frame_start, frame_end).tolist() + assert len(frame_indices) == T, ( + f"frame_indices length {len(frame_indices)} != T {T}, likely due to video decoder using different fps, i.e. sample with stride. Need to return frame indices from video decoder." + ) + all_masks = np.zeros((num_masks, T, H, W)).astype(np.uint8) + for idx, mid in enumerate(mask_ids_select): + mask = data_dict["segmentation"][mid] + shape = mask["segmentation_mask_rle"]["mask_shape"] + num_byte_per_mb = 1024 * 1024 + # total number of elements in uint8 (1 byte) / num_byte_per_mb + if shape[0] * shape[1] * shape[2] / num_byte_per_mb > self.thres_mb_python_decode: + # Switch to python decode if the mask is too large to avoid out of shared memory + + rle = decode_partial_rle_width1( + mask["segmentation_mask_rle"]["data"], + frame_start * shape[1] * shape[2], + frame_end * shape[1] * shape[2], + ) + partial_shape = (frame_end - frame_start, shape[1], shape[2]) + rle = rle.reshape(partial_shape) * 255 + else: + rle = pycocotools.mask.decode(mask["segmentation_mask_rle"]["data"]) + rle = rle.reshape(shape) * 255 + # Select the frames that are in the video + rle = np.stack([rle[i] for i in frame_indices]) + all_masks[idx] = rle + del rle + + key_out = self.output_keys[0] + # both value in [0,255] + # control_input_segmentation is the colored segmentation mask, value in [0,255], shape (3, T, H, W) + data_dict[key_out] = torch.from_numpy(segmentation_color_mask(all_masks, self.use_fixed_color_list)) + del all_masks # free memory + # Note: return this in trainnig will cause out of shared memory, uncomment only for data visualization + # control_input_segmentation_raw is the raw segmentation mask, value in {0,255}, shape (num_masks, T, H, W) + # all_masks_pad = np.zeros((self.num_masks, T, H, W)) + # all_masks_pad[: len(all_masks)] = all_masks + # data_dict[key_out + "_raw"] = torch.from_numpy(all_masks_pad) + return data_dict + + +class AddControlInputMask(Augmentor): + """Add control input masks for video inpainting (replace) and outpainting operations.""" + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_mask"], + thres_mb_python_decode: Optional[int] = 256, + args: Optional[dict] = None, + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + self.masking_params = args.get("masking_params", {}) + self.thres_mb_python_decode = thres_mb_python_decode + + def generate_video_outpaint_mask(self, T: int, H: int, W: int) -> torch.Tensor: + """Generate outpainting mask for video.""" + mask = np.ones((T, H, W), dtype=np.uint8) + mask_sub_type = self.masking_params.get("mask_sub_type") + + if mask_sub_type == "one_side": + region_range = self.masking_params["outpaint_region_range"]["one_side"] + n = int(H * np.random.uniform(region_range.min, region_range.max)) + direction = np.random.choice(["top", "bottom", "left", "right"]) + + for t in range(T): + if direction == "top": + mask[t, :n] = 0 + elif direction == "bottom": + mask[t, -n:] = 0 + elif direction == "left": + mask[t, :, :n] = 0 + else: + mask[t, :, -n:] = 0 + + elif mask_sub_type == "two_sides": + region_range = self.masking_params["outpaint_region_range"]["two_sides"] + n = int(H * np.random.uniform(region_range.min, region_range.max)) + direction = np.random.choice(["vertical", "horizontal"]) + + for t in range(T): + if direction == "vertical": + mask[t, :n] = mask[t, -n:] = 0 + else: + mask[t, :, :n] = mask[t, :, -n:] = 0 + + else: + region_range = self.masking_params["outpaint_region_range"]["all_sides"] + nw = int(W * np.random.uniform(region_range.min, region_range.max)) + nh = int(H * np.random.uniform(region_range.min, region_range.max)) + + mask_pos = mask_sub_type + # Calculate positions once before the frame loop + box_w = W - nw * 2 + box_h = H - nh * 2 + offset_h, offset_w = None, None + if mask_pos == "corner": + offset_h = np.random.choice([0, (H - box_h)]) + offset_w = np.random.choice([0, (W - box_w)]) + elif mask_pos == "random": + offset_h = np.random.randint(0, H - box_h) + offset_w = np.random.randint(0, W - box_w) + + # Apply same mask position to all frames + for t in range(T): + if mask_pos == "center": + mask[t, :nh] = mask[t, -nh:] = 0 + mask[t, :, :nw] = mask[t, :, -nw:] = 0 + else: + mask[t] = 0 + mask[t, offset_h : offset_h + box_h, offset_w : offset_w + box_w] = 1 + + return torch.from_numpy(mask) + + def generate_video_replace_mask( + self, frames: torch.Tensor, segmentation: list, frame_start: int, frame_end: int + ) -> torch.Tensor: + """Generate replacement mask for video.""" + _, T, H, W = frames.shape + obj_idx = None + mask = np.zeros((T, H, W), dtype=np.uint8) + + if segmentation: + obj_idx, obj_mask = self.get_object_mask(segmentation, frame_start, frame_end, T, H, W) + if obj_idx is not None: + mask = 1 - obj_mask + + return obj_idx, torch.from_numpy(mask) + + def decode_sam_masks(self, mask_data, frame_start, frame_end, shape) -> np.ndarray: + """Decode SAM RLE masks for video frames.""" + num_byte_per_mb = 1024 * 1024 + if shape[0] * shape[1] * shape[2] / num_byte_per_mb > self.thres_mb_python_decode: + rle = decode_partial_rle_width1( + mask_data, + frame_start * shape[1] * shape[2], + frame_end * shape[1] * shape[2], + ) + partial_shape = (frame_end - frame_start, shape[1], shape[2]) + rle = rle.reshape(partial_shape) + else: + rle = pycocotools.mask.decode(mask_data) + rle = rle.reshape(shape) + frame_indices = np.arange(frame_start, frame_end).tolist() + rle = np.stack([rle[i] for i in frame_indices]) + return rle + + def get_object_mask( + self, segmentation, frame_start, frame_end, T, H, W + ) -> tuple[Optional[int], Optional[np.ndarray]]: + """Get a valid object mask from video segmentation data using quality criteria. + + Args: + segmentation: List of dicts containing phrase and RLE mask data + frame_start: Starting frame index + frame_end: Ending frame index + T: Number of frames + H: Frame height + W: Frame width + + Returns: + tuple: (selected_idx, mask) where mask is a binary numpy array + """ + if not segmentation: + return None, None + + candidate_obj_indices = [] + candidate_masks = [] + total_frame_area = H * W + + temporal_presence_thre = self.masking_params["inpaint"].get("temporal_presence_thre", 0.2) + frame_quality_score_thre = self.masking_params["inpaint"].get("frame_quality_score_thre", 0.2) + + for obj_idx, seg in enumerate(segmentation): + if "segmentation_mask_rle" not in seg: + continue + + # Decode mask + shape = seg["segmentation_mask_rle"]["mask_shape"] + mask = self.decode_sam_masks(seg["segmentation_mask_rle"]["data"], frame_start, frame_end, shape) + mask = (mask > 0).astype(np.uint8) + if mask is None: + continue + + # Calculate temporal consistency + temporal_presence = np.mean([np.any(mask[t]) for t in range(T)]) + if temporal_presence < temporal_presence_thre: # Skip objects that aren't consistently present + continue + + # Evaluate mask quality for each frame + frame_qualities = [] + + for t in range(T): + cur_obj_mask = mask[t] + if not np.any(cur_obj_mask): + continue + + mask_area = cur_obj_mask.sum() + area_fraction_in_image = mask_area / total_frame_area + + # Get bounding box for the mask + y_indices, x_indices = np.where(cur_obj_mask) + if len(y_indices) == 0: + continue + box = [np.min(x_indices), np.min(y_indices), np.max(x_indices), np.max(y_indices)] + box_area = (box[2] - box[0]) * (box[3] - box[1]) + area_fraction_in_bbox = mask_area / box_area if box_area > 0 else 0 + + # Check quality criteria + enough_box_area = area_fraction_in_bbox > self.masking_params["inpaint"]["area_fraction_in_bbox_thre"] + enough_image_area = ( + area_fraction_in_image > self.masking_params["inpaint"]["area_fraction_in_image_thre"] + ) + + if "area_fraction_in_image_ceil" in self.masking_params["inpaint"]: + not_too_much_image_area = ( + area_fraction_in_image < self.masking_params["inpaint"]["area_fraction_in_image_ceil"] + ) + else: + not_too_much_image_area = True + + # Check border conditions + if "touch_border_thre" in self.masking_params["inpaint"]: + is_foreground, _ = check_if_foreground( + cur_obj_mask, border_threshold=self.masking_params["inpaint"]["touch_border_thre"] + ) + else: + is_foreground = True + + if "close_to_border_thre" in self.masking_params["inpaint"]: + not_close_to_border = check_within_border_thres( + cur_obj_mask, border_threshold=self.masking_params["inpaint"]["close_to_border_thre"] + ) + else: + not_close_to_border = True + + # All criteria must be met + if ( + enough_box_area + and is_foreground + and enough_image_area + and not_close_to_border + and not_too_much_image_area + ): + frame_qualities.append(1) + else: + frame_qualities.append(0) + + # Object is a candidate if it meets criteria in more than frame_quality_score_thre proportion of frames + frame_quality_score = np.mean(frame_qualities) if frame_qualities else 0 + if frame_quality_score > frame_quality_score_thre: + candidate_obj_indices.append(obj_idx) + candidate_masks.append(mask) + + if not candidate_obj_indices: + return None, None + # Randomly select from candidates that passed all criteria + selected_idx = np.random.randint(len(candidate_obj_indices)) + return candidate_obj_indices[selected_idx], candidate_masks[selected_idx] + + def sample_mask_type(self) -> tuple[str, str]: + """Sample mask type and subtype based on configured probabilities.""" + ratios = self.masking_params["task_ratio"] + + # Flatten probabilities into list of (type, subtype, prob) + choices = [] + for mask_type, subtypes in ratios.items(): + if mask_type == "outpaint": + for subtype, prob in subtypes.items(): + if subtype == "all_sides": + for pos, pos_prob in prob.items(): + choices.append((mask_type, pos, pos_prob)) + else: + choices.append((mask_type, subtype, prob)) + else: + for subtype, prob in subtypes.items(): + choices.append((mask_type, subtype, prob)) + + # Normalize probabilities + probs = np.array([c[2] for c in choices]) + probs = probs / probs.sum() + + # Sample type and subtype + idx = np.random.choice(len(choices), p=probs) + return choices[idx][0], choices[idx][1] + + def __call__(self, data_dict: dict) -> dict: + if "video" not in data_dict: + image = np.array(data_dict[self.input_keys[1]]) + h, w, _ = image.shape + data_dict[self.output_keys[0]] = torch.from_numpy(np.zeros((3, h, w)).astype(np.uint8)) + return data_dict + + frames = data_dict["video"] + _, T, H, W = frames.shape + frame_start = data_dict["frame_start"] + frame_end = data_dict["frame_end"] + + mask_type, mask_sub_type = self.sample_mask_type() + self.masking_params["mask_type"] = mask_type + self.masking_params["mask_sub_type"] = mask_sub_type + + if mask_type == "outpaint": + mask = self.generate_video_outpaint_mask(T, H, W) + data_dict["mask_phrase"] = "" + else: # replace or inpaint + selected_idx, mask = self.generate_video_replace_mask( + frames, data_dict["segmentation"], frame_start, frame_end + ) + if selected_idx is not None: + # Store selected object phrase + selected_phrase = data_dict["segmentation"][selected_idx]["phrase"] + data_dict["mask_phrase"] = selected_phrase + else: + data_dict["mask_phrase"] = "" + # Expand mask to 3 channels + mask_tensor = mask.unsqueeze(0).expand(3, -1, -1, -1) + data_dict[self.output_keys[0]] = mask_tensor + data_dict["mask_type"] = mask_type + data_dict["mask_sub_type"] = mask_sub_type + + return data_dict + + +class AddControlInputUpscale(Augmentor): + """ + Add control input to the data dictionary. control input are expanded to 3-channels + steps to add new items: modify this file, configs/conditioner.py, conditioner.py + """ + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_upscale"], + args: Optional[dict] = None, + use_random: bool = True, + preset_strength="medium", + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + self.use_random = use_random + self.preset_strength = preset_strength + + def __call__(self, data_dict: dict, target_size: tuple = None) -> dict: + if "control_input_upscale" in data_dict: + # already processed + return data_dict + key_img = self.input_keys[1] + key_out = self.output_keys[0] + frames = data_dict[key_img] + frames = np.array(frames) # CTHW + is_image = len(frames.shape) < 4 + if is_image: + frames = frames.transpose((2, 0, 1))[:, None] + h, w = frames.shape[-2:] + frames = torch.from_numpy(frames.transpose(1, 0, 2, 3)) # TCHW + + if "__url__" in data_dict and "aspect_ratio" in data_dict["__url__"].meta.opts: + aspect_ratio = data_dict["__url__"].meta.opts["aspect_ratio"] + elif "aspect_ratio" in data_dict: # Non-webdataset format + aspect_ratio = data_dict["aspect_ratio"] + else: + aspect_ratio = "16,9" + + # Define the crop size + RES_SIZE_INFO = IMAGE_RES_SIZE_INFO if is_image else VIDEO_RES_SIZE_INFO + crop_width, crop_height = RES_SIZE_INFO["720"][aspect_ratio] + + if self.use_random: # always on for training, always off for inference + # During training, randomly crop a patch, then randomly downsize the video and resize it back. + # Determine a random crop location + top = torch.randint(0, max(0, h - crop_height) + 1, (1,)).item() + left = torch.randint(0, max(0, w - crop_width) + 1, (1,)).item() + cropped_frames = frames[:, :, top : top + crop_height, left : left + crop_width] + + # Randomly downsample the video + # for 360p, 720p, 1080p -> 4k + scaler = np.random.choice([1 / 6, 1 / 3, 0.5], p=[0.3, 0.5, 0.2]) + small_crop_width = int(crop_width * scaler) + small_crop_height = int(crop_height * scaler) + resized_frames = transforms_F.resize( + cropped_frames, + size=(small_crop_height, small_crop_width), + interpolation=transforms_F.InterpolationMode.BICUBIC, + antialias=True, + ) + # Upsample to target size + resized_frames = transforms_F.resize( + resized_frames, + size=(crop_height, crop_width), + interpolation=transforms_F.InterpolationMode.BILINEAR, + ) + else: + if target_size is None: # for validation + # During validation, center crop a patch, then resize to target size. + if self.preset_strength == "low": + scaler = 0.5 + elif self.preset_strength == "medium": + scaler = 1 / 3 + else: + scaler = 1 / 6 + small_crop_width = int(crop_width * scaler) + small_crop_height = int(crop_height * scaler) + + # Center crop during inference + top = (h - small_crop_height) // 2 + left = (w - small_crop_width) // 2 + + # Perform the crop + frames = frames[:, :, top : top + small_crop_height, left : left + small_crop_width] + # Upsample to target size + resized_frames = transforms_F.resize( + frames, + size=(crop_height, crop_width), + interpolation=transforms_F.InterpolationMode.BILINEAR, + ) + else: # for inference + # During inference, directly resize to target size. + new_h, new_w = target_size + resized_frames = transforms_F.resize( + frames, + size=(new_h, new_w), + interpolation=transforms_F.InterpolationMode.BILINEAR, + ) + cropped_frames = resized_frames + + resized_frames = resized_frames.permute(1, 0, 2, 3).contiguous() # CTHW + cropped_frames = cropped_frames.permute(1, 0, 2, 3).contiguous() # CTHW + + if is_image: + resized_frames = resized_frames[:, 0] + cropped_frames = cropped_frames[:, 0] + data_dict[key_out] = resized_frames + data_dict[key_img] = cropped_frames + return data_dict + + +class AddControlInputHumanKpts(Augmentor): + """ + Add control input to the data dictionary. control input are expanded to 3-channels + steps to add new items: modify this file, configs/conditioner.py, conditioner.py + """ + + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = ["control_input_human_kpts"], + args: Optional[dict] = None, + **kwargs, + ) -> None: + super().__init__(input_keys, output_keys, args) + self.control_key_names = ["body-keypoints", "hand-keypoints"] + + def denormalize_pose_kpts(self, pose_kps: np.ndarray, h: int, w: int): + """ + pose_kps has shape = (#keypoints, 2) + or (#keypoints, 3) where the last dim is the confidence score. + """ + if pose_kps is not None: + out = pose_kps * np.array([w, h, 1]) + return out + else: + return None + + def draw_skeleton( + self, + img: np.ndarray, + keypoints: np.ndarray, + scores: np.ndarray, + kpt_thr: float = 0.5, + radius: int = 2, + line_width: int = 2, + ): + assert len(keypoints.shape) == 2 + keypoint_info, skeleton_info = ( + coco_wholebody_133_skeleton["keypoint_info"], + coco_wholebody_133_skeleton["skeleton_info"], + ) + vis_kpt = [s >= kpt_thr for s in scores] + + link_dict = {} + for i, kpt_info in keypoint_info.items(): + kpt_color = tuple(kpt_info["color"]) + link_dict[kpt_info["name"]] = kpt_info["id"] + + kpt = keypoints[i] + + if vis_kpt[i]: + img = cv2.circle(img, (int(kpt[0]), int(kpt[1])), int(radius), kpt_color, -1) + + for i, ske_info in skeleton_info.items(): + link = ske_info["link"] + pt0, pt1 = link_dict[link[0]], link_dict[link[1]] + + if vis_kpt[pt0] and vis_kpt[pt1]: + link_color = ske_info["color"] + kpt0 = keypoints[pt0] + kpt1 = keypoints[pt1] + + img = cv2.line( + img, (int(kpt0[0]), int(kpt0[1])), (int(kpt1[0]), int(kpt1[1])), link_color, thickness=line_width + ) + return img + + def plot_kpt_img(self, kpts_np_dict: dict, h: int, w: int) -> torch.Tensor: + """ + The raw human keypoint annotation are numpy arrays of pixel coordinates of the joints. + This function plots the keypoints on a black background to form a 3-channel image compatible with controlnet. + + Args: + kpts_np_dict (dict): A dict of keypoint annotations. Each value is a frame's annotation (a list of per-person dict). + H (int): height of the image + W (int): width of the image + Returns: + np.ndarray: keypoints of plotted on black background, shape = (3, T, H, W) + """ + + def plot_person_kpts(person_dict: dict, pose_vis_img: np.ndarray, h: int, w: int) -> np.ndarray: + """in-place update the pose image""" + body_keypoints = self.denormalize_pose_kpts(person_dict.get("body-keypoints"), h, w) + hand_keypoints = self.denormalize_pose_kpts(person_dict.get("hand-keypoints"), h, w) + assert body_keypoints is not None and hand_keypoints is not None, ( + "Both body and hand keypoints must be present." + ) + # all_keypoints: shape=(133, 3). following coco-fullbody skeleton config. 3 channels are x, y, confidence + all_keypoints = np.vstack([body_keypoints, hand_keypoints]) + kpts, scores = all_keypoints[..., :2], all_keypoints[..., -1] + pose_vis_img = self.draw_skeleton(pose_vis_img, kpts, scores, kpt_thr=0.5) + return pose_vis_img + + all_pose_img_frames = [] + # TODO: now loops over frames and persons. Check if this is a bottle net and try to optimize. + for t, kpts_np_frame in kpts_np_dict.items(): + pose_vis_img = np.zeros([h, w, 3]) + + # add each person's keypoints to this frame's pose image + for person_dict in kpts_np_frame: + plot_person_kpts(person_dict, pose_vis_img, h, w) # (h, w, 3) + all_pose_img_frames.append(pose_vis_img) + + pose_vis_vid = np.stack(all_pose_img_frames).transpose((3, 0, 1, 2)).astype(np.uint8) + return pose_vis_vid + + def extend_annotations_for_vid_chunk(self, annotation_dict: dict, total_frames: int) -> dict: + """ + For legacy data the annotations are done for chunks of every N frames (N=4). + This function repeats each chunk's first frame annotation to the rest frames + so that they become 'per-frame' and are ControlNet compatible. + + If the data is already per-frame annotated, then no need to call this. + Args: + annotation_dict (dict): Original annotations annotated every chunk_size frames. + Each value is a list of dicts, where each dict has many + human attributes. Here we only keep keypoint-relevant keys. + total_frames (int): Total number of frames in the video. + + Returns: + dict: extended annotations for all frames. + """ + annotated_frame_idxs = sorted(list(annotation_dict.keys())) + chunk_size = annotated_frame_idxs[1] - annotated_frame_idxs[0] # typically 1 or 4 + if chunk_size == 1: + # each person's dict can contain many irrelevant annotations (e.g. seg masks), here we only keep pose kpts + annotation_dict_simpler = { + key: [{k: v for k, v in sub_dict.items() if k in self.control_key_names} for sub_dict in sub_list] + for key, sub_list in annotation_dict.items() + } + return annotation_dict_simpler + + extended_annotations = {} + annotated_frames = sorted(annotation_dict.keys()) + + for start_frame in annotated_frames: + current_annotation = annotation_dict[start_frame] + end_frame = min(start_frame + chunk_size, total_frames) + for frame in range(start_frame, end_frame): + extended_annotations[frame] = [] + for person in current_annotation: + person_annotation = {} + for control_key in self.control_key_names: + person_annotation[control_key] = np.copy(person[control_key]) + extended_annotations[frame].append(person_annotation) + return extended_annotations + + def __call__(self, data_dict: dict) -> dict: + """ + human_annotations: loaded human annotation data pickle. One annotation per N frames. + In the past we did N=4; for ControlNet data annotations we will do N=1. + The pickle is a dict of annotated frames. + The key is the frame number. For each frame, as there can be multiple people, we maintain a list of per-person + dicts. Example: + { + 0: [ + {'body-keypoints': , 'hand-keypoints': }, + {'body-keypoints': , 'hand-keypoints': }, + ], # frame 0, 2 people + 4: [ + {'body-keypoints': , 'hand-keypoints': }, + {'body-keypoints': , 'hand-keypoints': }, + {'body-keypoints': , 'hand-keypoints': }, + ], # frame 4, 3 people + ... + } + Note that for the same person, their idx in the per-frame list isn't guaranteed to be consistent. + """ + human_annotations = data_dict.pop("human_annotation") + + frames = data_dict["video"] + _, T, H, W = frames.shape + + # same dict format as `human_annotations` but now every frame has an annotation + kpts_nparray_dict = self.extend_annotations_for_vid_chunk(human_annotations, T) + + # Colored human keypoints figure on black background. All persons in the same frame are plotted together. Shape: [3, T, H, W]. + kpts_cond_img = self.plot_kpt_img(kpts_nparray_dict, H, W) + + key_out = self.output_keys[0] + data_dict[key_out] = torch.from_numpy(kpts_cond_img) + return data_dict + + +if __name__ == "__main__": + import sys + + from cosmos1.models.diffusion.config.ctrl.augmentors import ( + BilateralOnlyBlurAugmentorConfig, + GaussianOnlyBlurAugmentorConfig, + ) + from cosmos1.models.diffusion.inference.demo_video import save_video + from cosmos1.models.diffusion.utils.inference_long_video import read_video_or_image_into_frames_BCTHW + + path_in = sys.argv[1] + + def main(input_file_path: str) -> None: + max_length = 10 + video_input = read_video_or_image_into_frames_BCTHW(input_file_path, normalize=False)[0, :, :max_length] + C, T, H, W = video_input.shape + blur_processes = { + "bilateral": BilateralOnlyBlurAugmentorConfig, + "gaussian": GaussianOnlyBlurAugmentorConfig, + } + for blur_name, blur_process in blur_processes.items(): + for preset_strength in ["low", "medium", "high"]: + process = get_augmentor_for_eval( + "video", + "control_input_blur", + preset_strength=preset_strength, + blur_config=blur_process[preset_strength], + ) + output = process({"video": video_input}) + output = output["control_input_blur"].numpy().transpose((1, 2, 3, 0)) + + output_file_path = f"{input_file_path[:-4]}_{blur_name}_{preset_strength}.mp4" + save_video( + grid=output, + fps=5, + H=H, + W=W, + video_save_quality=9, + video_save_path=output_file_path, + ) + + print(f"Output video saved as {output_file_path}") + + main(path_in) + +# flake8: noqa: F821 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/guided_filter.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/guided_filter.py new file mode 100644 index 00000000..c3a66533 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/guided_filter.py @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +# -*- coding: utf-8 -*- +# @package guided_filter.core.filters +# +# Implementation of guided filter. +# * GuidedFilter: Original guided filter. +# * FastGuidedFilter: Fast version of the guided filter. +# @author tody +# @date 2015/08/26 + +import cv2 +import numpy as np + + +# Convert image into float32 type. +def to32F(img): + if img.dtype == np.float32: + return img + return (1.0 / 255.0) * np.float32(img) + + +# Convert image into uint8 type. +def to8U(img): + if img.dtype == np.uint8: + return img + return np.clip(np.uint8(255.0 * img), 0, 255) + + +# Return if the input image is gray or not. +def _isGray(I): + return len(I.shape) == 2 + + +# Return down sampled image. +# @param scale (w/s, h/s) image will be created. +# @param shape I.shape[:2]=(h, w). numpy friendly size parameter. +def _downSample(I, scale=4, shape=None): + if shape is not None: + h, w = shape + return cv2.resize(I, (w, h), interpolation=cv2.INTER_NEAREST) + + h, w = I.shape[:2] + return cv2.resize(I, (int(w / scale), int(h / scale)), interpolation=cv2.INTER_NEAREST) + + +# Return up sampled image. +# @param scale (w*s, h*s) image will be created. +# @param shape I.shape[:2]=(h, w). numpy friendly size parameter. +def _upSample(I, scale=2, shape=None): + if shape is not None: + h, w = shape + return cv2.resize(I, (w, h), interpolation=cv2.INTER_LINEAR) + + h, w = I.shape[:2] + return cv2.resize(I, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_LINEAR) + + +# Fast guide filter. +class FastGuidedFilter: + # Constructor. + # @param I Input guidance image. Color or gray. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + # @param scale Down sampled scale. + def __init__(self, I, radius=5, epsilon=0.4, scale=4): + I_32F = to32F(I) + self._I = I_32F + h, w = I.shape[:2] + + I_sub = _downSample(I_32F, scale) + + self._I_sub = I_sub + radius = int(radius / scale) + + if _isGray(I): + self._guided_filter = GuidedFilterGray(I_sub, radius, epsilon) + else: + self._guided_filter = GuidedFilterColor(I_sub, radius, epsilon) + + # Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + p_32F = to32F(p) + shape_original = p.shape[:2] + + p_sub = _downSample(p_32F, shape=self._I_sub.shape[:2]) + + if _isGray(p_sub): + return self._filterGray(p_sub, shape_original) + + cs = p.shape[2] + q = np.array(p_32F) + + for ci in range(cs): + q[:, :, ci] = self._filterGray(p_sub[:, :, ci], shape_original) + return to8U(q) + + def _filterGray(self, p_sub, shape_original): + ab_sub = self._guided_filter._computeCoefficients(p_sub) + ab = [_upSample(abi, shape=shape_original) for abi in ab_sub] + return self._guided_filter._computeOutput(ab, self._I) + + +# Guide filter. +class GuidedFilter: + # Constructor. + # @param I Input guidance image. Color or gray. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + def __init__(self, I, radius=5, epsilon=0.4): + I_32F = to32F(I) + + if _isGray(I): + self._guided_filter = GuidedFilterGray(I_32F, radius, epsilon) + else: + self._guided_filter = GuidedFilterColor(I_32F, radius, epsilon) + + # Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + return to8U(self._guided_filter.filter(p)) + + +# Common parts of guided filter. +# +# This class is used by guided_filter class. GuidedFilterGray and GuidedFilterColor. +# Based on guided_filter._computeCoefficients, guided_filter._computeOutput, +# GuidedFilterCommon.filter computes filtered image for color and gray. +class GuidedFilterCommon: + def __init__(self, guided_filter): + self._guided_filter = guided_filter + + # Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + p_32F = to32F(p) + if _isGray(p_32F): + return self._filterGray(p_32F) + + cs = p.shape[2] + q = np.array(p_32F) + + for ci in range(cs): + q[:, :, ci] = self._filterGray(p_32F[:, :, ci]) + return q + + def _filterGray(self, p): + ab = self._guided_filter._computeCoefficients(p) + return self._guided_filter._computeOutput(ab, self._guided_filter._I) + + +# Guided filter for gray guidance image. +class GuidedFilterGray: + # @param I Input gray guidance image. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + def __init__(self, I, radius=5, epsilon=0.4): + self._radius = 2 * radius + 1 + self._epsilon = epsilon + self._I = to32F(I) + self._initFilter() + self._filter_common = GuidedFilterCommon(self) + + # Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + return self._filter_common.filter(p) + + def _initFilter(self): + I = self._I + r = self._radius + self._I_mean = cv2.blur(I, (r, r)) + I_mean_sq = cv2.blur(I**2, (r, r)) + self._I_var = I_mean_sq - self._I_mean**2 + + def _computeCoefficients(self, p): + r = self._radius + p_mean = cv2.blur(p, (r, r)) + p_cov = p_mean - self._I_mean * p_mean + a = p_cov / (self._I_var + self._epsilon) + b = p_mean - a * self._I_mean + a_mean = cv2.blur(a, (r, r)) + b_mean = cv2.blur(b, (r, r)) + return a_mean, b_mean + + def _computeOutput(self, ab, I): + a_mean, b_mean = ab + return a_mean * I + b_mean + + +# Guided filter for color guidance image. +class GuidedFilterColor: + # @param I Input color guidance image. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + def __init__(self, I, radius=5, epsilon=0.2): + self._radius = 2 * radius + 1 + self._epsilon = epsilon + self._I = to32F(I) + self._initFilter() + self._filter_common = GuidedFilterCommon(self) + + # Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + return self._filter_common.filter(p) + + def _initFilter(self): + I = self._I + r = self._radius + eps = self._epsilon + + Ir, Ig, Ib = I[:, :, 0], I[:, :, 1], I[:, :, 2] + + self._Ir_mean = cv2.blur(Ir, (r, r)) + self._Ig_mean = cv2.blur(Ig, (r, r)) + self._Ib_mean = cv2.blur(Ib, (r, r)) + + Irr_var = cv2.blur(Ir**2, (r, r)) - self._Ir_mean**2 + eps + Irg_var = cv2.blur(Ir * Ig, (r, r)) - self._Ir_mean * self._Ig_mean + Irb_var = cv2.blur(Ir * Ib, (r, r)) - self._Ir_mean * self._Ib_mean + Igg_var = cv2.blur(Ig * Ig, (r, r)) - self._Ig_mean * self._Ig_mean + eps + Igb_var = cv2.blur(Ig * Ib, (r, r)) - self._Ig_mean * self._Ib_mean + Ibb_var = cv2.blur(Ib * Ib, (r, r)) - self._Ib_mean * self._Ib_mean + eps + + Irr_inv = Igg_var * Ibb_var - Igb_var * Igb_var + Irg_inv = Igb_var * Irb_var - Irg_var * Ibb_var + Irb_inv = Irg_var * Igb_var - Igg_var * Irb_var + Igg_inv = Irr_var * Ibb_var - Irb_var * Irb_var + Igb_inv = Irb_var * Irg_var - Irr_var * Igb_var + Ibb_inv = Irr_var * Igg_var - Irg_var * Irg_var + + I_cov = Irr_inv * Irr_var + Irg_inv * Irg_var + Irb_inv * Irb_var + Irr_inv /= I_cov + Irg_inv /= I_cov + Irb_inv /= I_cov + Igg_inv /= I_cov + Igb_inv /= I_cov + Ibb_inv /= I_cov + + self._Irr_inv = Irr_inv + self._Irg_inv = Irg_inv + self._Irb_inv = Irb_inv + self._Igg_inv = Igg_inv + self._Igb_inv = Igb_inv + self._Ibb_inv = Ibb_inv + + def _computeCoefficients(self, p): + r = self._radius + I = self._I + Ir, Ig, Ib = I[:, :, 0], I[:, :, 1], I[:, :, 2] + + p_mean = cv2.blur(p, (r, r)) + + Ipr_mean = cv2.blur(Ir * p, (r, r)) + Ipg_mean = cv2.blur(Ig * p, (r, r)) + Ipb_mean = cv2.blur(Ib * p, (r, r)) + + Ipr_cov = Ipr_mean - self._Ir_mean * p_mean + Ipg_cov = Ipg_mean - self._Ig_mean * p_mean + Ipb_cov = Ipb_mean - self._Ib_mean * p_mean + + ar = self._Irr_inv * Ipr_cov + self._Irg_inv * Ipg_cov + self._Irb_inv * Ipb_cov + ag = self._Irg_inv * Ipr_cov + self._Igg_inv * Ipg_cov + self._Igb_inv * Ipb_cov + ab = self._Irb_inv * Ipr_cov + self._Igb_inv * Ipg_cov + self._Ibb_inv * Ipb_cov + b = p_mean - ar * self._Ir_mean - ag * self._Ig_mean - ab * self._Ib_mean + + ar_mean = cv2.blur(ar, (r, r)) + ag_mean = cv2.blur(ag, (r, r)) + ab_mean = cv2.blur(ab, (r, r)) + b_mean = cv2.blur(b, (r, r)) + + return ar_mean, ag_mean, ab_mean, b_mean + + def _computeOutput(self, ab, I): + ar_mean, ag_mean, ab_mean, b_mean = ab + + Ir, Ig, Ib = I[:, :, 0], I[:, :, 1], I[:, :, 2] + + q = ar_mean * Ir + ag_mean * Ig + ab_mean * Ib + b_mean + + return q + + +# flake8: noqa: E741 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/human_keypoint_utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/human_keypoint_utils.py new file mode 100644 index 00000000..6bd83975 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/datasets/augmentors/human_keypoint_utils.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Adapted from rtmlib: + https://raw.githubusercontent.com/Tau-J/rtmlib/refs/heads/main/rtmlib/visualization/skeleton/coco133.py +""" + +coco_wholebody_133_skeleton = dict( + name="coco133", + keypoint_info={ + 0: dict(name="nose", id=0, color=[51, 153, 255], swap=""), + 1: dict(name="left_eye", id=1, color=[51, 153, 255], swap="right_eye"), + 2: dict(name="right_eye", id=2, color=[51, 153, 255], swap="left_eye"), + 3: dict(name="left_ear", id=3, color=[51, 153, 255], swap="right_ear"), + 4: dict(name="right_ear", id=4, color=[51, 153, 255], swap="left_ear"), + 5: dict(name="left_shoulder", id=5, color=[0, 255, 0], swap="right_shoulder"), + 6: dict(name="right_shoulder", id=6, color=[255, 128, 0], swap="left_shoulder"), + 7: dict(name="left_elbow", id=7, color=[0, 255, 0], swap="right_elbow"), + 8: dict(name="right_elbow", id=8, color=[255, 128, 0], swap="left_elbow"), + 9: dict(name="left_wrist", id=9, color=[0, 255, 0], swap="right_wrist"), + 10: dict(name="right_wrist", id=10, color=[255, 128, 0], swap="left_wrist"), + 11: dict(name="left_hip", id=11, color=[0, 255, 0], swap="right_hip"), + 12: dict(name="right_hip", id=12, color=[255, 128, 0], swap="left_hip"), + 13: dict(name="left_knee", id=13, color=[0, 255, 0], swap="right_knee"), + 14: dict(name="right_knee", id=14, color=[255, 128, 0], swap="left_knee"), + 15: dict(name="left_ankle", id=15, color=[0, 255, 0], swap="right_ankle"), + 16: dict(name="right_ankle", id=16, color=[255, 128, 0], swap="left_ankle"), + 17: dict(name="left_big_toe", id=17, color=[255, 128, 0], swap="right_big_toe"), + 18: dict(name="left_small_toe", id=18, color=[255, 128, 0], swap="right_small_toe"), + 19: dict(name="left_heel", id=19, color=[255, 128, 0], swap="right_heel"), + 20: dict(name="right_big_toe", id=20, color=[255, 128, 0], swap="left_big_toe"), + 21: dict(name="right_small_toe", id=21, color=[255, 128, 0], swap="left_small_toe"), + 22: dict(name="right_heel", id=22, color=[255, 128, 0], swap="left_heel"), + 23: dict(name="face-0", id=23, color=[255, 255, 255], swap="face-16"), + 24: dict(name="face-1", id=24, color=[255, 255, 255], swap="face-15"), + 25: dict(name="face-2", id=25, color=[255, 255, 255], swap="face-14"), + 26: dict(name="face-3", id=26, color=[255, 255, 255], swap="face-13"), + 27: dict(name="face-4", id=27, color=[255, 255, 255], swap="face-12"), + 28: dict(name="face-5", id=28, color=[255, 255, 255], swap="face-11"), + 29: dict(name="face-6", id=29, color=[255, 255, 255], swap="face-10"), + 30: dict(name="face-7", id=30, color=[255, 255, 255], swap="face-9"), + 31: dict(name="face-8", id=31, color=[255, 255, 255], swap=""), + 32: dict(name="face-9", id=32, color=[255, 255, 255], swap="face-7"), + 33: dict(name="face-10", id=33, color=[255, 255, 255], swap="face-6"), + 34: dict(name="face-11", id=34, color=[255, 255, 255], swap="face-5"), + 35: dict(name="face-12", id=35, color=[255, 255, 255], swap="face-4"), + 36: dict(name="face-13", id=36, color=[255, 255, 255], swap="face-3"), + 37: dict(name="face-14", id=37, color=[255, 255, 255], swap="face-2"), + 38: dict(name="face-15", id=38, color=[255, 255, 255], swap="face-1"), + 39: dict(name="face-16", id=39, color=[255, 255, 255], swap="face-0"), + 40: dict(name="face-17", id=40, color=[255, 255, 255], swap="face-26"), + 41: dict(name="face-18", id=41, color=[255, 255, 255], swap="face-25"), + 42: dict(name="face-19", id=42, color=[255, 255, 255], swap="face-24"), + 43: dict(name="face-20", id=43, color=[255, 255, 255], swap="face-23"), + 44: dict(name="face-21", id=44, color=[255, 255, 255], swap="face-22"), + 45: dict(name="face-22", id=45, color=[255, 255, 255], swap="face-21"), + 46: dict(name="face-23", id=46, color=[255, 255, 255], swap="face-20"), + 47: dict(name="face-24", id=47, color=[255, 255, 255], swap="face-19"), + 48: dict(name="face-25", id=48, color=[255, 255, 255], swap="face-18"), + 49: dict(name="face-26", id=49, color=[255, 255, 255], swap="face-17"), + 50: dict(name="face-27", id=50, color=[255, 255, 255], swap=""), + 51: dict(name="face-28", id=51, color=[255, 255, 255], swap=""), + 52: dict(name="face-29", id=52, color=[255, 255, 255], swap=""), + 53: dict(name="face-30", id=53, color=[255, 255, 255], swap=""), + 54: dict(name="face-31", id=54, color=[255, 255, 255], swap="face-35"), + 55: dict(name="face-32", id=55, color=[255, 255, 255], swap="face-34"), + 56: dict(name="face-33", id=56, color=[255, 255, 255], swap=""), + 57: dict(name="face-34", id=57, color=[255, 255, 255], swap="face-32"), + 58: dict(name="face-35", id=58, color=[255, 255, 255], swap="face-31"), + 59: dict(name="face-36", id=59, color=[255, 255, 255], swap="face-45"), + 60: dict(name="face-37", id=60, color=[255, 255, 255], swap="face-44"), + 61: dict(name="face-38", id=61, color=[255, 255, 255], swap="face-43"), + 62: dict(name="face-39", id=62, color=[255, 255, 255], swap="face-42"), + 63: dict(name="face-40", id=63, color=[255, 255, 255], swap="face-47"), + 64: dict(name="face-41", id=64, color=[255, 255, 255], swap="face-46"), + 65: dict(name="face-42", id=65, color=[255, 255, 255], swap="face-39"), + 66: dict(name="face-43", id=66, color=[255, 255, 255], swap="face-38"), + 67: dict(name="face-44", id=67, color=[255, 255, 255], swap="face-37"), + 68: dict(name="face-45", id=68, color=[255, 255, 255], swap="face-36"), + 69: dict(name="face-46", id=69, color=[255, 255, 255], swap="face-41"), + 70: dict(name="face-47", id=70, color=[255, 255, 255], swap="face-40"), + 71: dict(name="face-48", id=71, color=[255, 255, 255], swap="face-54"), + 72: dict(name="face-49", id=72, color=[255, 255, 255], swap="face-53"), + 73: dict(name="face-50", id=73, color=[255, 255, 255], swap="face-52"), + 74: dict(name="face-51", id=74, color=[255, 255, 255], swap=""), + 75: dict(name="face-52", id=75, color=[255, 255, 255], swap="face-50"), + 76: dict(name="face-53", id=76, color=[255, 255, 255], swap="face-49"), + 77: dict(name="face-54", id=77, color=[255, 255, 255], swap="face-48"), + 78: dict(name="face-55", id=78, color=[255, 255, 255], swap="face-59"), + 79: dict(name="face-56", id=79, color=[255, 255, 255], swap="face-58"), + 80: dict(name="face-57", id=80, color=[255, 255, 255], swap=""), + 81: dict(name="face-58", id=81, color=[255, 255, 255], swap="face-56"), + 82: dict(name="face-59", id=82, color=[255, 255, 255], swap="face-55"), + 83: dict(name="face-60", id=83, color=[255, 255, 255], swap="face-64"), + 84: dict(name="face-61", id=84, color=[255, 255, 255], swap="face-63"), + 85: dict(name="face-62", id=85, color=[255, 255, 255], swap=""), + 86: dict(name="face-63", id=86, color=[255, 255, 255], swap="face-61"), + 87: dict(name="face-64", id=87, color=[255, 255, 255], swap="face-60"), + 88: dict(name="face-65", id=88, color=[255, 255, 255], swap="face-67"), + 89: dict(name="face-66", id=89, color=[255, 255, 255], swap=""), + 90: dict(name="face-67", id=90, color=[255, 255, 255], swap="face-65"), + 91: dict(name="left_hand_root", id=91, color=[255, 255, 255], swap="right_hand_root"), + 92: dict(name="left_thumb1", id=92, color=[255, 128, 0], swap="right_thumb1"), + 93: dict(name="left_thumb2", id=93, color=[255, 128, 0], swap="right_thumb2"), + 94: dict(name="left_thumb3", id=94, color=[255, 128, 0], swap="right_thumb3"), + 95: dict(name="left_thumb4", id=95, color=[255, 128, 0], swap="right_thumb4"), + 96: dict(name="left_forefinger1", id=96, color=[255, 153, 255], swap="right_forefinger1"), + 97: dict(name="left_forefinger2", id=97, color=[255, 153, 255], swap="right_forefinger2"), + 98: dict(name="left_forefinger3", id=98, color=[255, 153, 255], swap="right_forefinger3"), + 99: dict(name="left_forefinger4", id=99, color=[255, 153, 255], swap="right_forefinger4"), + 100: dict(name="left_middle_finger1", id=100, color=[102, 178, 255], swap="right_middle_finger1"), + 101: dict(name="left_middle_finger2", id=101, color=[102, 178, 255], swap="right_middle_finger2"), + 102: dict(name="left_middle_finger3", id=102, color=[102, 178, 255], swap="right_middle_finger3"), + 103: dict(name="left_middle_finger4", id=103, color=[102, 178, 255], swap="right_middle_finger4"), + 104: dict(name="left_ring_finger1", id=104, color=[255, 51, 51], swap="right_ring_finger1"), + 105: dict(name="left_ring_finger2", id=105, color=[255, 51, 51], swap="right_ring_finger2"), + 106: dict(name="left_ring_finger3", id=106, color=[255, 51, 51], swap="right_ring_finger3"), + 107: dict(name="left_ring_finger4", id=107, color=[255, 51, 51], swap="right_ring_finger4"), + 108: dict(name="left_pinky_finger1", id=108, color=[0, 255, 0], swap="right_pinky_finger1"), + 109: dict(name="left_pinky_finger2", id=109, color=[0, 255, 0], swap="right_pinky_finger2"), + 110: dict(name="left_pinky_finger3", id=110, color=[0, 255, 0], swap="right_pinky_finger3"), + 111: dict(name="left_pinky_finger4", id=111, color=[0, 255, 0], swap="right_pinky_finger4"), + 112: dict(name="right_hand_root", id=112, color=[255, 255, 255], swap="left_hand_root"), + 113: dict(name="right_thumb1", id=113, color=[255, 128, 0], swap="left_thumb1"), + 114: dict(name="right_thumb2", id=114, color=[255, 128, 0], swap="left_thumb2"), + 115: dict(name="right_thumb3", id=115, color=[255, 128, 0], swap="left_thumb3"), + 116: dict(name="right_thumb4", id=116, color=[255, 128, 0], swap="left_thumb4"), + 117: dict(name="right_forefinger1", id=117, color=[255, 153, 255], swap="left_forefinger1"), + 118: dict(name="right_forefinger2", id=118, color=[255, 153, 255], swap="left_forefinger2"), + 119: dict(name="right_forefinger3", id=119, color=[255, 153, 255], swap="left_forefinger3"), + 120: dict(name="right_forefinger4", id=120, color=[255, 153, 255], swap="left_forefinger4"), + 121: dict(name="right_middle_finger1", id=121, color=[102, 178, 255], swap="left_middle_finger1"), + 122: dict(name="right_middle_finger2", id=122, color=[102, 178, 255], swap="left_middle_finger2"), + 123: dict(name="right_middle_finger3", id=123, color=[102, 178, 255], swap="left_middle_finger3"), + 124: dict(name="right_middle_finger4", id=124, color=[102, 178, 255], swap="left_middle_finger4"), + 125: dict(name="right_ring_finger1", id=125, color=[255, 51, 51], swap="left_ring_finger1"), + 126: dict(name="right_ring_finger2", id=126, color=[255, 51, 51], swap="left_ring_finger2"), + 127: dict(name="right_ring_finger3", id=127, color=[255, 51, 51], swap="left_ring_finger3"), + 128: dict(name="right_ring_finger4", id=128, color=[255, 51, 51], swap="left_ring_finger4"), + 129: dict(name="right_pinky_finger1", id=129, color=[0, 255, 0], swap="left_pinky_finger1"), + 130: dict(name="right_pinky_finger2", id=130, color=[0, 255, 0], swap="left_pinky_finger2"), + 131: dict(name="right_pinky_finger3", id=131, color=[0, 255, 0], swap="left_pinky_finger3"), + 132: dict(name="right_pinky_finger4", id=132, color=[0, 255, 0], swap="left_pinky_finger4"), + }, + skeleton_info={ + 0: dict(link=("left_ankle", "left_knee"), id=0, color=[0, 255, 0]), + 1: dict(link=("left_knee", "left_hip"), id=1, color=[0, 255, 0]), + 2: dict(link=("right_ankle", "right_knee"), id=2, color=[255, 128, 0]), + 3: dict(link=("right_knee", "right_hip"), id=3, color=[255, 128, 0]), + 4: dict(link=("left_hip", "right_hip"), id=4, color=[51, 153, 255]), + 5: dict(link=("left_shoulder", "left_hip"), id=5, color=[51, 153, 255]), + 6: dict(link=("right_shoulder", "right_hip"), id=6, color=[51, 153, 255]), + 7: dict(link=("left_shoulder", "right_shoulder"), id=7, color=[51, 153, 255]), + 8: dict(link=("left_shoulder", "left_elbow"), id=8, color=[0, 255, 0]), + 9: dict(link=("right_shoulder", "right_elbow"), id=9, color=[255, 128, 0]), + 10: dict(link=("left_elbow", "left_wrist"), id=10, color=[0, 255, 0]), + 11: dict(link=("right_elbow", "right_wrist"), id=11, color=[255, 128, 0]), + 12: dict(link=("left_eye", "right_eye"), id=12, color=[51, 153, 255]), + 13: dict(link=("nose", "left_eye"), id=13, color=[51, 153, 255]), + 14: dict(link=("nose", "right_eye"), id=14, color=[51, 153, 255]), + 15: dict(link=("left_eye", "left_ear"), id=15, color=[51, 153, 255]), + 16: dict(link=("right_eye", "right_ear"), id=16, color=[51, 153, 255]), + 17: dict(link=("left_ear", "left_shoulder"), id=17, color=[51, 153, 255]), + 18: dict(link=("right_ear", "right_shoulder"), id=18, color=[51, 153, 255]), + 19: dict(link=("left_ankle", "left_big_toe"), id=19, color=[0, 255, 0]), + 20: dict(link=("left_ankle", "left_small_toe"), id=20, color=[0, 255, 0]), + 21: dict(link=("left_ankle", "left_heel"), id=21, color=[0, 255, 0]), + 22: dict(link=("right_ankle", "right_big_toe"), id=22, color=[255, 128, 0]), + 23: dict(link=("right_ankle", "right_small_toe"), id=23, color=[255, 128, 0]), + 24: dict(link=("right_ankle", "right_heel"), id=24, color=[255, 128, 0]), + 25: dict(link=("left_hand_root", "left_thumb1"), id=25, color=[255, 128, 0]), + 26: dict(link=("left_thumb1", "left_thumb2"), id=26, color=[255, 128, 0]), + 27: dict(link=("left_thumb2", "left_thumb3"), id=27, color=[255, 128, 0]), + 28: dict(link=("left_thumb3", "left_thumb4"), id=28, color=[255, 128, 0]), + 29: dict(link=("left_hand_root", "left_forefinger1"), id=29, color=[255, 153, 255]), + 30: dict(link=("left_forefinger1", "left_forefinger2"), id=30, color=[255, 153, 255]), + 31: dict(link=("left_forefinger2", "left_forefinger3"), id=31, color=[255, 153, 255]), + 32: dict(link=("left_forefinger3", "left_forefinger4"), id=32, color=[255, 153, 255]), + 33: dict(link=("left_hand_root", "left_middle_finger1"), id=33, color=[102, 178, 255]), + 34: dict(link=("left_middle_finger1", "left_middle_finger2"), id=34, color=[102, 178, 255]), + 35: dict(link=("left_middle_finger2", "left_middle_finger3"), id=35, color=[102, 178, 255]), + 36: dict(link=("left_middle_finger3", "left_middle_finger4"), id=36, color=[102, 178, 255]), + 37: dict(link=("left_hand_root", "left_ring_finger1"), id=37, color=[255, 51, 51]), + 38: dict(link=("left_ring_finger1", "left_ring_finger2"), id=38, color=[255, 51, 51]), + 39: dict(link=("left_ring_finger2", "left_ring_finger3"), id=39, color=[255, 51, 51]), + 40: dict(link=("left_ring_finger3", "left_ring_finger4"), id=40, color=[255, 51, 51]), + 41: dict(link=("left_hand_root", "left_pinky_finger1"), id=41, color=[0, 255, 0]), + 42: dict(link=("left_pinky_finger1", "left_pinky_finger2"), id=42, color=[0, 255, 0]), + 43: dict(link=("left_pinky_finger2", "left_pinky_finger3"), id=43, color=[0, 255, 0]), + 44: dict(link=("left_pinky_finger3", "left_pinky_finger4"), id=44, color=[0, 255, 0]), + 45: dict(link=("right_hand_root", "right_thumb1"), id=45, color=[255, 128, 0]), + 46: dict(link=("right_thumb1", "right_thumb2"), id=46, color=[255, 128, 0]), + 47: dict(link=("right_thumb2", "right_thumb3"), id=47, color=[255, 128, 0]), + 48: dict(link=("right_thumb3", "right_thumb4"), id=48, color=[255, 128, 0]), + 49: dict(link=("right_hand_root", "right_forefinger1"), id=49, color=[255, 153, 255]), + 50: dict(link=("right_forefinger1", "right_forefinger2"), id=50, color=[255, 153, 255]), + 51: dict(link=("right_forefinger2", "right_forefinger3"), id=51, color=[255, 153, 255]), + 52: dict(link=("right_forefinger3", "right_forefinger4"), id=52, color=[255, 153, 255]), + 53: dict(link=("right_hand_root", "right_middle_finger1"), id=53, color=[102, 178, 255]), + 54: dict(link=("right_middle_finger1", "right_middle_finger2"), id=54, color=[102, 178, 255]), + 55: dict(link=("right_middle_finger2", "right_middle_finger3"), id=55, color=[102, 178, 255]), + 56: dict(link=("right_middle_finger3", "right_middle_finger4"), id=56, color=[102, 178, 255]), + 57: dict(link=("right_hand_root", "right_ring_finger1"), id=57, color=[255, 51, 51]), + 58: dict(link=("right_ring_finger1", "right_ring_finger2"), id=58, color=[255, 51, 51]), + 59: dict(link=("right_ring_finger2", "right_ring_finger3"), id=59, color=[255, 51, 51]), + 60: dict(link=("right_ring_finger3", "right_ring_finger4"), id=60, color=[255, 51, 51]), + 61: dict(link=("right_hand_root", "right_pinky_finger1"), id=61, color=[0, 255, 0]), + 62: dict(link=("right_pinky_finger1", "right_pinky_finger2"), id=62, color=[0, 255, 0]), + 63: dict(link=("right_pinky_finger2", "right_pinky_finger3"), id=63, color=[0, 255, 0]), + 64: dict(link=("right_pinky_finger3", "right_pinky_finger4"), id=64, color=[0, 255, 0]), + }, +) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/batch_ops.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/batch_ops.py new file mode 100644 index 00000000..62b4de0d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/batch_ops.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Functions for performing operations with broadcasting to the right axis +# +# Example +# input1: tensor of size (N1, N2) +# input2: tensor of size (N1, N2, N3, N4) +# batch_mul(input1, input2) = input1[:, :, None, None] * input2 +# +# If the common dimensions don't match, we raise an assertion error. + +# pylint: disable=C0115,C0116,C0301 + +from torch import Tensor + + +def common_broadcast(x: Tensor, y: Tensor) -> tuple[Tensor, Tensor]: + ndims1 = x.ndim + ndims2 = y.ndim + + common_ndims = min(ndims1, ndims2) + for axis in range(common_ndims): + assert x.shape[axis] == y.shape[axis], "Dimensions not equal at axis {}".format(axis) + + if ndims1 < ndims2: + x = x.reshape(x.shape + (1,) * (ndims2 - ndims1)) + elif ndims2 < ndims1: + y = y.reshape(y.shape + (1,) * (ndims1 - ndims2)) + + return x, y + + +def batch_mul(x: Tensor, y: Tensor) -> Tensor: + x, y = common_broadcast(x, y) + return x * y diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/multi_step.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/multi_step.py new file mode 100644 index 00000000..c66d3ba9 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/multi_step.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +Impl of multistep methods to solve the ODE in the diffusion model. +""" + +from typing import Callable, List, Tuple + +import torch +from cosmos1.models.diffusion.diffusion.functional.runge_kutta import reg_x0_euler_step, res_x0_rk2_step + + +def order2_fn( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_s: torch.Tensor, x0_preds: torch.Tensor +) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """ + impl the second order multistep method in https://arxiv.org/pdf/2308.02157 + Adams Bashforth approach! + """ + if x0_preds: + x0_s1, s1 = x0_preds[0] + x_t = res_x0_rk2_step(x_s, t, s, x0_s, s1, x0_s1) + else: + x_t = reg_x0_euler_step(x_s, s, t, x0_s)[0] + return x_t, [(x0_s, s)] + + +# key: method name, value: method function +# key: order + algorithm name +MULTISTEP_FNs = { + "2ab": order2_fn, +} + + +def get_multi_step_fn(name: str) -> Callable: + if name in MULTISTEP_FNs: + return MULTISTEP_FNs[name] + methods = "\n\t".join(MULTISTEP_FNs.keys()) + raise RuntimeError("Only support multistep method\n" + methods) + + +def is_multi_step_fn_supported(name: str) -> bool: + """ + Check if the multistep method is supported. + """ + return name in MULTISTEP_FNs diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/runge_kutta.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/runge_kutta.py new file mode 100644 index 00000000..66c9448b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/functional/runge_kutta.py @@ -0,0 +1,332 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Tuple + +import torch +from cosmos1.models.diffusion.diffusion.functional.batch_ops import batch_mul + + +def phi1(t: torch.Tensor) -> torch.Tensor: + """ + Compute the first order phi function: (exp(t) - 1) / t. + + Args: + t: Input tensor. + + Returns: + Tensor: Result of phi1 function. + """ + input_dtype = t.dtype + t = t.to(dtype=torch.float64) + return (torch.expm1(t) / t).to(dtype=input_dtype) + + +def phi2(t: torch.Tensor) -> torch.Tensor: + """ + Compute the second order phi function: (phi1(t) - 1) / t. + + Args: + t: Input tensor. + + Returns: + Tensor: Result of phi2 function. + """ + input_dtype = t.dtype + t = t.to(dtype=torch.float64) + return ((phi1(t) - 1.0) / t).to(dtype=input_dtype) + + +def res_x0_rk2_step( + x_s: torch.Tensor, + t: torch.Tensor, + s: torch.Tensor, + x0_s: torch.Tensor, + s1: torch.Tensor, + x0_s1: torch.Tensor, +) -> torch.Tensor: + """ + Perform a residual-based 2nd order Runge-Kutta step. + + Args: + x_s: Current state tensor. + t: Target time tensor. + s: Current time tensor. + x0_s: Prediction at current time. + s1: Intermediate time tensor. + x0_s1: Prediction at intermediate time. + + Returns: + Tensor: Updated state tensor. + + Raises: + AssertionError: If step size is too small. + """ + s = -torch.log(s) + t = -torch.log(t) + m = -torch.log(s1) + + dt = t - s + assert not torch.any(torch.isclose(dt, torch.zeros_like(dt), atol=1e-6)), "Step size is too small" + assert not torch.any(torch.isclose(m - s, torch.zeros_like(dt), atol=1e-6)), "Step size is too small" + + c2 = (m - s) / dt + phi1_val, phi2_val = phi1(-dt), phi2(-dt) + + # Handle edge case where t = s = m + b1 = torch.nan_to_num(phi1_val - 1.0 / c2 * phi2_val, nan=0.0) + b2 = torch.nan_to_num(1.0 / c2 * phi2_val, nan=0.0) + + return batch_mul(torch.exp(-dt), x_s) + batch_mul(dt, batch_mul(b1, x0_s) + batch_mul(b2, x0_s1)) + + +def reg_x0_euler_step( + x_s: torch.Tensor, + s: torch.Tensor, + t: torch.Tensor, + x0_s: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a regularized Euler step based on x0 prediction. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_s: Prediction at current time. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current prediction. + """ + coef_x0 = (s - t) / s + coef_xs = t / s + return batch_mul(coef_x0, x0_s) + batch_mul(coef_xs, x_s), x0_s + + +def reg_eps_euler_step( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, eps_s: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a regularized Euler step based on epsilon prediction. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + eps_s: Epsilon prediction at current time. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current x0 prediction. + """ + return x_s + batch_mul(eps_s, t - s), x_s + batch_mul(eps_s, 0 - s) + + +def rk1_euler( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a first-order Runge-Kutta (Euler) step. + + Recommended for diffusion models with guidance or model undertrained + Usually more stable at the cost of a bit slower convergence. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and x0 prediction. + """ + x0_s = x0_fn(x_s, s) + return reg_x0_euler_step(x_s, s, t, x0_s) + + +def rk2_mid_stable( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a stable second-order Runge-Kutta (midpoint) step. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and x0 prediction. + """ + s1 = torch.sqrt(s * t) + x_s1, _ = rk1_euler(x_s, s, s1, x0_fn) + + x0_s1 = x0_fn(x_s1, s1) + return reg_x0_euler_step(x_s, s, t, x0_s1) + + +def rk2_mid(x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a second-order Runge-Kutta (midpoint) step. + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and x0 prediction. + """ + s1 = torch.sqrt(s * t) + x_s1, x0_s = rk1_euler(x_s, s, s1, x0_fn) + + x0_s1 = x0_fn(x_s1, s1) + + return res_x0_rk2_step(x_s, t, s, x0_s, s1, x0_s1), x0_s1 + + +def rk_2heun_naive( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a naive second-order Runge-Kutta (Heun's method) step. + Impl based on rho-rk-deis solvers, https://github.com/qsh-zh/deis + Recommended for diffusion models without guidance and relative large NFE + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current state. + """ + x_t, x0_s = rk1_euler(x_s, s, t, x0_fn) + eps_s = batch_mul(1.0 / s, x_t - x0_s) + x0_t = x0_fn(x_t, t) + eps_t = batch_mul(1.0 / t, x_t - x0_t) + + avg_eps = (eps_s + eps_t) / 2 + + return reg_eps_euler_step(x_s, s, t, avg_eps) + + +def rk_2heun_edm( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a naive second-order Runge-Kutta (Heun's method) step. + Impl based no EDM second order Heun method + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current state. + """ + x_t, x0_s = rk1_euler(x_s, s, t, x0_fn) + x0_t = x0_fn(x_t, t) + + avg_x0 = (x0_s + x0_t) / 2 + + return reg_x0_euler_step(x_s, s, t, avg_x0) + + +def rk_3kutta_naive( + x_s: torch.Tensor, s: torch.Tensor, t: torch.Tensor, x0_fn: Callable +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Perform a naive third-order Runge-Kutta step. + Impl based on rho-rk-deis solvers, https://github.com/qsh-zh/deis + Recommended for diffusion models without guidance and relative large NFE + + Args: + x_s: Current state tensor. + s: Current time tensor. + t: Target time tensor. + x0_fn: Function to compute x0 prediction. + + Returns: + Tuple[Tensor, Tensor]: Updated state tensor and current state. + """ + c2, c3 = 0.5, 1.0 + a31, a32 = -1.0, 2.0 + b1, b2, b3 = 1.0 / 6, 4.0 / 6, 1.0 / 6 + + delta = t - s + + s1 = c2 * delta + s + s2 = c3 * delta + s + x_s1, x0_s = rk1_euler(x_s, s, s1, x0_fn) + eps_s = batch_mul(1.0 / s, x_s - x0_s) + x0_s1 = x0_fn(x_s1, s1) + eps_s1 = batch_mul(1.0 / s1, x_s1 - x0_s1) + + _eps = a31 * eps_s + a32 * eps_s1 + x_s2, _ = reg_eps_euler_step(x_s, s, s2, _eps) + + x0_s2 = x0_fn(x_s2, s2) + eps_s2 = batch_mul(1.0 / s2, x_s2 - x0_s2) + + avg_eps = b1 * eps_s + b2 * eps_s1 + b3 * eps_s2 + return reg_eps_euler_step(x_s, s, t, avg_eps) + + +# key : order + name +RK_FNs = { + "1euler": rk1_euler, + "2mid": rk2_mid, + "2mid_stable": rk2_mid_stable, + "2heun_edm": rk_2heun_edm, + "2heun_naive": rk_2heun_naive, + "3kutta_naive": rk_3kutta_naive, +} + + +def get_runge_kutta_fn(name: str) -> Callable: + """ + Get the specified Runge-Kutta function. + + Args: + name: Name of the Runge-Kutta method. + + Returns: + Callable: The specified Runge-Kutta function. + + Raises: + RuntimeError: If the specified method is not supported. + """ + if name in RK_FNs: + return RK_FNs[name] + methods = "\n\t".join(RK_FNs.keys()) + raise RuntimeError(f"Only support the following Runge-Kutta methods:\n\t{methods}") + + +def is_runge_kutta_fn_supported(name: str) -> bool: + """ + Check if the specified Runge-Kutta function is supported. + + Args: + name: Name of the Runge-Kutta method. + + Returns: + bool: True if the method is supported, False otherwise. + """ + return name in RK_FNs diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/modules/denoiser_scaling.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/modules/denoiser_scaling.py new file mode 100644 index 00000000..588e6c74 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/modules/denoiser_scaling.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Tuple + +import torch + + +class EDMScaling: + def __init__(self, sigma_data: float = 0.5): + self.sigma_data = sigma_data + + def __call__(self, sigma: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + c_skip = self.sigma_data**2 / (sigma**2 + self.sigma_data**2) + c_out = sigma * self.sigma_data / (sigma**2 + self.sigma_data**2) ** 0.5 + c_in = 1 / (sigma**2 + self.sigma_data**2) ** 0.5 + c_noise = 0.25 * sigma.log() + return c_skip, c_out, c_in, c_noise diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/modules/res_sampler.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/modules/res_sampler.py new file mode 100644 index 00000000..d5ae5422 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/modules/res_sampler.py @@ -0,0 +1,285 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +A general framework for various sampling algorithm from a diffusion model. +Impl based on +* Refined Exponential Solver (RES) in https://arxiv.org/pdf/2308.02157 +* also clude other impl, DDIM, DEIS, DPM-Solver, EDM sampler. +Most of sampling algorihtm, Runge-Kutta, Multi-step, etc, can be impl in this framework by \ + adding new step function in get_runge_kutta_fn or get_multi_step_fn. +""" + +import math +from typing import Any, Callable, List, Literal, Optional, Tuple, Union + +import attrs +import torch +from cosmos1.models.diffusion.diffusion.functional.multi_step import get_multi_step_fn, is_multi_step_fn_supported +from cosmos1.models.diffusion.diffusion.functional.runge_kutta import get_runge_kutta_fn, is_runge_kutta_fn_supported +from cosmos1.utils.config import make_freezable + + +COMMON_SOLVER_OPTIONS = Literal["2ab", "2mid", "1euler"] + + +@make_freezable +@attrs.define(slots=False) +class SolverConfig: + is_multi: bool = False + rk: str = "2mid" + multistep: str = "2ab" + # following parameters control stochasticity, see EDM paper + # BY default, we use deterministic with no stochasticity + s_churn: float = 0.0 + s_t_max: float = float("inf") + s_t_min: float = 0.05 + s_noise: float = 1.0 + + +@make_freezable +@attrs.define(slots=False) +class SolverTimestampConfig: + nfe: int = 50 + t_min: float = 0.002 + t_max: float = 80.0 + order: float = 7.0 + is_forward: bool = False # whether generate forward or backward timestamps + + +@make_freezable +@attrs.define(slots=False) +class SamplerConfig: + solver: SolverConfig = attrs.field(factory=SolverConfig) + timestamps: SolverTimestampConfig = attrs.field(factory=SolverTimestampConfig) + sample_clean: bool = True # whether run one last step to generate clean image + + +def get_rev_ts( + t_min: float, t_max: float, num_steps: int, ts_order: Union[int, float], is_forward: bool = False +) -> torch.Tensor: + """ + Generate a sequence of reverse time steps. + + Args: + t_min (float): The minimum time value. + t_max (float): The maximum time value. + num_steps (int): The number of time steps to generate. + ts_order (Union[int, float]): The order of the time step progression. + is_forward (bool, optional): If True, returns the sequence in forward order. Defaults to False. + + Returns: + torch.Tensor: A tensor containing the generated time steps in reverse or forward order. + + Raises: + ValueError: If `t_min` is not less than `t_max`. + TypeError: If `ts_order` is not an integer or float. + """ + if t_min >= t_max: + raise ValueError("t_min must be less than t_max") + + if not isinstance(ts_order, (int, float)): + raise TypeError("ts_order must be an integer or float") + + step_indices = torch.arange(num_steps + 1, dtype=torch.float64) + time_steps = ( + t_max ** (1 / ts_order) + step_indices / num_steps * (t_min ** (1 / ts_order) - t_max ** (1 / ts_order)) + ) ** ts_order + + if is_forward: + return time_steps.flip(dims=(0,)) + + return time_steps + + +class Sampler(torch.nn.Module): + def __init__(self, cfg: Optional[SamplerConfig] = None): + super().__init__() + if cfg is None: + cfg = SamplerConfig() + self.cfg = cfg + + @torch.no_grad() + def forward( + self, + x0_fn: Callable, + x_sigma_max: torch.Tensor, + num_steps: int = 35, + sigma_min: float = 0.002, + sigma_max: float = 80, + rho: float = 7, + S_churn: float = 0, + S_min: float = 0, + S_max: float = float("inf"), + S_noise: float = 1, + solver_option: str = "2ab", + ) -> torch.Tensor: + in_dtype = x_sigma_max.dtype + + def float64_x0_fn(x_B_StateShape: torch.Tensor, t_B: torch.Tensor) -> torch.Tensor: + return x0_fn(x_B_StateShape.to(in_dtype), t_B.to(in_dtype)).to(torch.float64) + + is_multistep = is_multi_step_fn_supported(solver_option) + is_rk = is_runge_kutta_fn_supported(solver_option) + assert is_multistep or is_rk, f"Only support multistep or Runge-Kutta method, got {solver_option}" + + solver_cfg = SolverConfig( + s_churn=S_churn, + s_t_max=S_max, + s_t_min=S_min, + s_noise=S_noise, + is_multi=is_multistep, + rk=solver_option, + multistep=solver_option, + ) + timestamps_cfg = SolverTimestampConfig(nfe=num_steps, t_min=sigma_min, t_max=sigma_max, order=rho) + sampler_cfg = SamplerConfig(solver=solver_cfg, timestamps=timestamps_cfg, sample_clean=True) + + return self._forward_impl(float64_x0_fn, x_sigma_max, sampler_cfg).to(in_dtype) + + @torch.no_grad() + def _forward_impl( + self, + denoiser_fn: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], + noisy_input_B_StateShape: torch.Tensor, + sampler_cfg: Optional[SamplerConfig] = None, + callback_fns: Optional[List[Callable]] = None, + ) -> torch.Tensor: + """ + Internal implementation of the forward pass. + + Args: + denoiser_fn: Function to denoise the input. + noisy_input_B_StateShape: Input tensor with noise. + sampler_cfg: Configuration for the sampler. + callback_fns: List of callback functions to be called during sampling. + + Returns: + torch.Tensor: Denoised output tensor. + """ + sampler_cfg = self.cfg if sampler_cfg is None else sampler_cfg + solver_order = 1 if sampler_cfg.solver.is_multi else int(sampler_cfg.solver.rk[0]) + num_timestamps = sampler_cfg.timestamps.nfe // solver_order + + sigmas_L = get_rev_ts( + sampler_cfg.timestamps.t_min, sampler_cfg.timestamps.t_max, num_timestamps, sampler_cfg.timestamps.order + ).to(noisy_input_B_StateShape.device) + + denoised_output = differential_equation_solver( + denoiser_fn, sigmas_L, sampler_cfg.solver, callback_fns=callback_fns + )(noisy_input_B_StateShape) + + if sampler_cfg.sample_clean: + # Override denoised_output with fully denoised version + ones = torch.ones(denoised_output.size(0), device=denoised_output.device, dtype=denoised_output.dtype) + denoised_output = denoiser_fn(denoised_output, sigmas_L[-1] * ones) + + return denoised_output + + +def fori_loop(lower: int, upper: int, body_fun: Callable[[int, Any], Any], init_val: Any) -> Any: + """ + Implements a for loop with a function. + + Args: + lower: Lower bound of the loop (inclusive). + upper: Upper bound of the loop (exclusive). + body_fun: Function to be applied in each iteration. + init_val: Initial value for the loop. + + Returns: + The final result after all iterations. + """ + val = init_val + for i in range(lower, upper): + val = body_fun(i, val) + return val + + +def differential_equation_solver( + x0_fn: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], + sigmas_L: torch.Tensor, + solver_cfg: SolverConfig, + callback_fns: Optional[List[Callable]] = None, +) -> Callable[[torch.Tensor], torch.Tensor]: + """ + Creates a differential equation solver function. + + Args: + x0_fn: Function to compute x0 prediction. + sigmas_L: Tensor of sigma values with shape [L,]. + solver_cfg: Configuration for the solver. + callback_fns: Optional list of callback functions. + + Returns: + A function that solves the differential equation. + """ + num_step = len(sigmas_L) - 1 + + if solver_cfg.is_multi: + update_step_fn = get_multi_step_fn(solver_cfg.multistep) + else: + update_step_fn = get_runge_kutta_fn(solver_cfg.rk) + + eta = min(solver_cfg.s_churn / (num_step + 1), math.sqrt(1.2) - 1) + + def sample_fn(input_xT_B_StateShape: torch.Tensor) -> torch.Tensor: + """ + Samples from the differential equation. + + Args: + input_xT_B_StateShape: Input tensor with shape [B, StateShape]. + + Returns: + Output tensor with shape [B, StateShape]. + """ + ones_B = torch.ones(input_xT_B_StateShape.size(0), device=input_xT_B_StateShape.device, dtype=torch.float64) + + def step_fn( + i_th: int, state: Tuple[torch.Tensor, Optional[List[torch.Tensor]]] + ) -> Tuple[torch.Tensor, Optional[List[torch.Tensor]]]: + input_x_B_StateShape, x0_preds = state + sigma_cur_0, sigma_next_0 = sigmas_L[i_th], sigmas_L[i_th + 1] + + # algorithm 2: line 4-6 + if solver_cfg.s_t_min < sigma_cur_0 < solver_cfg.s_t_max: + hat_sigma_cur_0 = sigma_cur_0 + eta * sigma_cur_0 + input_x_B_StateShape = input_x_B_StateShape + ( + hat_sigma_cur_0**2 - sigma_cur_0**2 + ).sqrt() * solver_cfg.s_noise * torch.randn_like(input_x_B_StateShape) + sigma_cur_0 = hat_sigma_cur_0 + + if solver_cfg.is_multi: + x0_pred_B_StateShape = x0_fn(input_x_B_StateShape, sigma_cur_0 * ones_B) + output_x_B_StateShape, x0_preds = update_step_fn( + input_x_B_StateShape, sigma_cur_0 * ones_B, sigma_next_0 * ones_B, x0_pred_B_StateShape, x0_preds + ) + else: + output_x_B_StateShape, x0_preds = update_step_fn( + input_x_B_StateShape, sigma_cur_0 * ones_B, sigma_next_0 * ones_B, x0_fn + ) + + if callback_fns: + for callback_fn in callback_fns: + callback_fn(**locals()) + + return output_x_B_StateShape, x0_preds + + x_at_eps, _ = fori_loop(0, num_step, step_fn, [input_xT_B_StateShape, None]) + return x_at_eps + + return sample_fn diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/types.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/types.py new file mode 100644 index 00000000..ac277cd9 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/diffusion/types.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import torch + + +@dataclass +class DenoisePrediction: + x0: torch.Tensor # clean data prediction + eps: Optional[torch.Tensor] = None # noise prediction + logvar: Optional[torch.Tensor] = None # log variance of noise prediction, can be used a confidence / uncertainty diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/inference_utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/inference_utils.py new file mode 100644 index 00000000..cad29027 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/inference_utils.py @@ -0,0 +1,829 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import importlib +from contextlib import contextmanager +from typing import List, NamedTuple, Optional, Tuple + +import einops +import imageio +import numpy as np +import torch +import torchvision.transforms.functional as transforms_F +from cosmos1.models.diffusion.model.model_sample_multiview_driving import DiffusionMultiCameraV2WModel +from cosmos1.models.diffusion.model.model_t2w import DiffusionT2WModel +from cosmos1.models.diffusion.model.model_v2w import DiffusionV2WModel +from cosmos1.utils import log, misc +from cosmos1.utils.config_helper import get_config_module, override +from cosmos1.utils.io import load_from_fileobj + + +TORCH_VERSION: Tuple[int, ...] = tuple(int(x) for x in torch.__version__.split(".")[:2]) +if TORCH_VERSION >= (1, 11): + from torch.ao import quantization + from torch.ao.quantization import FakeQuantizeBase, ObserverBase +elif ( + TORCH_VERSION >= (1, 8) + and hasattr(torch.quantization, "FakeQuantizeBase") + and hasattr(torch.quantization, "ObserverBase") +): + from torch import quantization + from torch.quantization import FakeQuantizeBase, ObserverBase + +DEFAULT_AUGMENT_SIGMA = 0.001 + + +def add_common_arguments(parser): + """Add common command line arguments for text2world and video2world generation. + + Args: + parser (ArgumentParser): Argument parser to add arguments to + + The arguments include: + - checkpoint_dir: Base directory containing model weights + - tokenizer_dir: Directory containing tokenizer weights + - video_save_name: Output video filename for single video generation + - video_save_folder: Output directory for batch video generation + - prompt: Text prompt for single video generation + - batch_input_path: Path to JSONL file with input prompts for batch video generation + - negative_prompt: Text prompt describing undesired attributes + - num_steps: Number of diffusion sampling steps + - guidance: Classifier-free guidance scale + - num_video_frames: Number of frames to generate + - height/width: Output video dimensions + - fps: Output video frame rate + - seed: Random seed for reproducibility + - Various model offloading flags + """ + parser.add_argument( + "--checkpoint_dir", type=str, default="checkpoints", help="Base directory containing model checkpoints" + ) + parser.add_argument( + "--tokenizer_dir", + type=str, + default="Cosmos-1.0-Tokenizer-CV8x8x8", + help="Tokenizer weights directory relative to checkpoint_dir", + ) + parser.add_argument( + "--video_save_name", + type=str, + default="output", + help="Output filename for generating a single video", + ) + parser.add_argument( + "--video_save_folder", + type=str, + default="outputs/", + help="Output folder for generating a batch of videos", + ) + parser.add_argument( + "--prompt", + type=str, + help="Text prompt for generating a single video", + ) + parser.add_argument( + "--batch_input_path", + type=str, + help="Path to a JSONL file of input prompts for generating a batch of videos", + ) + parser.add_argument( + "--negative_prompt", + type=str, + default="The video captures a series of frames showing ugly scenes, static with no motion, motion blur, " + "over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, " + "underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, " + "jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special " + "effects, fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and " + "flickering. Overall, the video is of poor quality.", + help="Negative prompt for the video", + ) + parser.add_argument("--num_steps", type=int, default=35, help="Number of diffusion sampling steps") + parser.add_argument("--guidance", type=float, default=7, help="Guidance scale value") + parser.add_argument( + "--num_video_frames", type=int, default=121, choices=[121], help="Number of video frames to sample" + ) + parser.add_argument("--height", type=int, default=704, help="Height of video to sample") + parser.add_argument("--width", type=int, default=1280, help="Width of video to sample") + parser.add_argument("--fps", type=int, default=24, help="FPS of the sampled video") + parser.add_argument("--seed", type=int, default=1, help="Random seed") + parser.add_argument( + "--disable_prompt_upsampler", + action="store_true", + help="Disable prompt upsampling", + ) + parser.add_argument( + "--offload_diffusion_transformer", + action="store_true", + help="Offload DiT after inference", + ) + parser.add_argument( + "--offload_tokenizer", + action="store_true", + help="Offload tokenizer after inference", + ) + parser.add_argument( + "--offload_text_encoder_model", + action="store_true", + help="Offload text encoder model after inference", + ) + parser.add_argument( + "--offload_prompt_upsampler", + action="store_true", + help="Offload prompt upsampler after inference", + ) + parser.add_argument( + "--offload_guardrail_models", + action="store_true", + help="Offload guardrail models after inference", + ) + + +# Function to fully remove an argument +def remove_argument(parser, arg_name): + # Get a list of actions to remove + actions_to_remove = [action for action in parser._actions if action.dest == arg_name] + + for action in actions_to_remove: + # Remove action from parser._actions + parser._actions.remove(action) + + # Remove option strings + for option_string in action.option_strings: + parser._option_string_actions.pop(option_string, None) + + +def validate_args(args: argparse.Namespace, inference_type: str) -> None: + """Validate command line arguments for text2world and video2world generation.""" + assert inference_type in [ + "text2world", + "video2world", + ], "Invalid inference_type, must be 'text2world' or 'video2world'" + + # Validate prompt/image/video args for single or batch generation + if inference_type == "text2world" or (inference_type == "video2world" and args.disable_prompt_upsampler): + assert args.prompt or args.batch_input_path, "--prompt or --batch_input_path must be provided." + if inference_type == "video2world" and not args.batch_input_path: + assert args.input_image_or_video_path, ( + "--input_image_or_video_path must be provided for single video generation." + ) + + +class _IncompatibleKeys( + NamedTuple( + "IncompatibleKeys", + [ + ("missing_keys", List[str]), + ("unexpected_keys", List[str]), + ("incorrect_shapes", List[Tuple[str, Tuple[int], Tuple[int]]]), + ], + ) +): + pass + + +def non_strict_load_model(model: torch.nn.Module, checkpoint_state_dict: dict) -> _IncompatibleKeys: + """Load a model checkpoint with non-strict matching, handling shape mismatches. + + Args: + model (torch.nn.Module): Model to load weights into + checkpoint_state_dict (dict): State dict from checkpoint + + Returns: + _IncompatibleKeys: Named tuple containing: + - missing_keys: Keys present in model but missing from checkpoint + - unexpected_keys: Keys present in checkpoint but not in model + - incorrect_shapes: Keys with mismatched tensor shapes + + The function handles special cases like: + - Uninitialized parameters + - Quantization observers + - TransformerEngine FP8 states + """ + # workaround https://github.com/pytorch/pytorch/issues/24139 + model_state_dict = model.state_dict() + incorrect_shapes = [] + for k in list(checkpoint_state_dict.keys()): + if k in model_state_dict: + if "_extra_state" in k: # Key introduced by TransformerEngine for FP8 + log.debug(f"Skipping key {k} introduced by TransformerEngine for FP8 in the checkpoint.") + continue + model_param = model_state_dict[k] + # Allow mismatch for uninitialized parameters + if TORCH_VERSION >= (1, 8) and isinstance(model_param, torch.nn.parameter.UninitializedParameter): + continue + if not isinstance(model_param, torch.Tensor): + raise ValueError( + f"Find non-tensor parameter {k} in the model. type: {type(model_param)} {type(checkpoint_state_dict[k])}, please check if this key is safe to skip or not." + ) + + shape_model = tuple(model_param.shape) + shape_checkpoint = tuple(checkpoint_state_dict[k].shape) + if shape_model != shape_checkpoint: + has_observer_base_classes = ( + TORCH_VERSION >= (1, 8) + and hasattr(quantization, "ObserverBase") + and hasattr(quantization, "FakeQuantizeBase") + ) + if has_observer_base_classes: + # Handle the special case of quantization per channel observers, + # where buffer shape mismatches are expected. + def _get_module_for_key(model: torch.nn.Module, key: str) -> torch.nn.Module: + # foo.bar.param_or_buffer_name -> [foo, bar] + key_parts = key.split(".")[:-1] + cur_module = model + for key_part in key_parts: + cur_module = getattr(cur_module, key_part) + return cur_module + + cls_to_skip = ( + ObserverBase, + FakeQuantizeBase, + ) + target_module = _get_module_for_key(model, k) + if isinstance(target_module, cls_to_skip): + # Do not remove modules with expected shape mismatches + # them from the state_dict loading. They have special logic + # in _load_from_state_dict to handle the mismatches. + continue + + incorrect_shapes.append((k, shape_checkpoint, shape_model)) + checkpoint_state_dict.pop(k) + incompatible = model.load_state_dict(checkpoint_state_dict, strict=False) + # Remove keys with "_extra_state" suffix, which are non-parameter items introduced by TransformerEngine for FP8 handling + missing_keys = [k for k in incompatible.missing_keys if "_extra_state" not in k] + unexpected_keys = [k for k in incompatible.unexpected_keys if "_extra_state" not in k] + return _IncompatibleKeys( + missing_keys=missing_keys, + unexpected_keys=unexpected_keys, + incorrect_shapes=incorrect_shapes, + ) + + +@contextmanager +def skip_init_linear(): + # skip init of nn.Linear + orig_reset_parameters = torch.nn.Linear.reset_parameters + torch.nn.Linear.reset_parameters = lambda x: x + xavier_uniform_ = torch.nn.init.xavier_uniform_ + torch.nn.init.xavier_uniform_ = lambda x: x + yield + torch.nn.Linear.reset_parameters = orig_reset_parameters + torch.nn.init.xavier_uniform_ = xavier_uniform_ + + +def load_model_by_config( + config_job_name, + config_file="projects/cosmos_video/config/config.py", + model_class=DiffusionT2WModel, +): + config_module = get_config_module(config_file) + config = importlib.import_module(config_module).make_config() + + config = override(config, ["--", f"experiment={config_job_name}"]) + + # Check that the config is valid + config.validate() + # Freeze the config so developers don't change it during training. + config.freeze() # type: ignore + + # Initialize model + with skip_init_linear(): + model = model_class(config.model) + return model + + +def load_network_model(model: DiffusionT2WModel, ckpt_path: str): + with skip_init_linear(): + model.set_up_model() + net_state_dict = torch.load(ckpt_path, map_location="cpu", weights_only=True) + log.debug(non_strict_load_model(model.model, net_state_dict)) + model.cuda() + + +def load_tokenizer_model(model: DiffusionT2WModel, tokenizer_dir: str): + with skip_init_linear(): + model.set_up_tokenizer(tokenizer_dir) + model.cuda() + + +def prepare_data_batch( + height: int, + width: int, + num_frames: int, + fps: int, + prompt_embedding: torch.Tensor, + negative_prompt_embedding: Optional[torch.Tensor] = None, +): + """Prepare input batch tensors for video generation. + + Args: + height (int): Height of video frames + width (int): Width of video frames + num_frames (int): Number of frames to generate + fps (int): Frames per second + prompt_embedding (torch.Tensor): Encoded text prompt embeddings + negative_prompt_embedding (torch.Tensor, optional): Encoded negative prompt embeddings + + Returns: + dict: Batch dictionary containing: + - video: Zero tensor of target video shape + - t5_text_mask: Attention mask for text embeddings + - image_size: Target frame dimensions + - fps: Target frame rate + - num_frames: Number of frames + - padding_mask: Frame padding mask + - t5_text_embeddings: Prompt embeddings + - neg_t5_text_embeddings: Negative prompt embeddings (if provided) + - neg_t5_text_mask: Mask for negative embeddings (if provided) + """ + # Create base data batch + data_batch = { + "video": torch.zeros((1, 3, num_frames, height, width), dtype=torch.uint8).cuda(), + "t5_text_mask": torch.ones(1, 512, dtype=torch.bfloat16).cuda(), + "image_size": torch.tensor([[height, width, height, width]] * 1, dtype=torch.bfloat16).cuda(), + "fps": torch.tensor([fps] * 1, dtype=torch.bfloat16).cuda(), + "num_frames": torch.tensor([num_frames] * 1, dtype=torch.bfloat16).cuda(), + "padding_mask": torch.zeros((1, 1, height, width), dtype=torch.bfloat16).cuda(), + } + + # Handle text embeddings + + t5_embed = prompt_embedding.to(dtype=torch.bfloat16).cuda() + data_batch["t5_text_embeddings"] = t5_embed + + if negative_prompt_embedding is not None: + neg_t5_embed = negative_prompt_embedding.to(dtype=torch.bfloat16).cuda() + data_batch["neg_t5_text_embeddings"] = neg_t5_embed + data_batch["neg_t5_text_mask"] = torch.ones(1, 512, dtype=torch.bfloat16).cuda() + + return data_batch + + +def get_video_batch(model, prompt_embedding, negative_prompt_embedding, height, width, fps, num_video_frames): + """Prepare complete input batch for video generation including latent dimensions. + + Args: + model: Diffusion model instance + prompt_embedding (torch.Tensor): Text prompt embeddings + negative_prompt_embedding (torch.Tensor): Negative prompt embeddings + height (int): Output video height + width (int): Output video width + fps (int): Output video frame rate + num_video_frames (int): Number of frames to generate + + Returns: + tuple: + - data_batch (dict): Complete model input batch + - state_shape (list): Shape of latent state [C,T,H,W] accounting for VAE compression + """ + raw_video_batch = prepare_data_batch( + height=height, + width=width, + num_frames=num_video_frames, + fps=fps, + prompt_embedding=prompt_embedding, + negative_prompt_embedding=negative_prompt_embedding, + ) + state_shape = [ + model.tokenizer.channel, + model.tokenizer.get_latent_num_frames(num_video_frames), + height // model.tokenizer.spatial_compression_factor, + width // model.tokenizer.spatial_compression_factor, + ] + return raw_video_batch, state_shape + + +def get_video_batch_for_multi_camera_model( + model, prompt_embedding, height, width, fps, num_video_frames, frame_repeat_negative_condition +): + """Prepare complete input batch for video generation including latent dimensions. + + Args: + model: Diffusion model instance + prompt_embedding (torch.Tensor): Text prompt embeddings + height (int): Output video height + width (int): Output video width + fps (int): Output video frame rate + num_video_frames (int): Number of frames to generate + frame_repeat_negative_condition (int): Number of frames to generate + + Returns: + tuple: + - data_batch (dict): Complete model input batch + - state_shape (list): Shape of latent state [C,T,H,W] accounting for VAE compression + """ + n_cameras = len(prompt_embedding) + prompt_embedding = einops.rearrange(torch.cat(prompt_embedding), "n t d -> (n t) d").unsqueeze(0) + raw_video_batch = prepare_data_batch( + height=height, + width=width, + num_frames=num_video_frames, + fps=fps, + prompt_embedding=prompt_embedding, + ) + if frame_repeat_negative_condition != -1: + frame_repeat = torch.zeros(n_cameras) + frame_repeat[-1] = frame_repeat_negative_condition + frame_repeat[-2] = frame_repeat_negative_condition + raw_video_batch["frame_repeat"] = frame_repeat.unsqueeze(0).to(dtype=torch.bfloat16).cuda() + state_shape = [ + model.tokenizer.channel, + model.tokenizer.get_latent_num_frames(int(num_video_frames / n_cameras)) * n_cameras, + height // model.tokenizer.spatial_compression_factor, + width // model.tokenizer.spatial_compression_factor, + ] + return raw_video_batch, state_shape + + +def generate_world_from_text( + model: DiffusionT2WModel, + state_shape: list[int], + is_negative_prompt: bool, + data_batch: dict, + guidance: float, + num_steps: int, + seed: int, +): + """Generate video from text prompt using diffusion model. + + Args: + model (DiffusionT2WModel): Text-to-video diffusion model + state_shape (list[int]): Latent state dimensions [C,T,H,W] + is_negative_prompt (bool): Whether negative prompt is provided + data_batch (dict): Model input batch with embeddings + guidance (float): Classifier-free guidance scale + num_steps (int): Number of diffusion sampling steps + seed (int): Random seed for reproducibility + + Returns: + np.ndarray: Generated video frames [T,H,W,C], range [0,255] + + The function: + 1. Initializes random latent with maximum noise + 2. Performs guided diffusion sampling + 3. Decodes latents to pixel space + """ + x_sigma_max = ( + misc.arch_invariant_rand( + (1,) + tuple(state_shape), + torch.float32, + model.tensor_kwargs["device"], + seed, + ) + * model.sde.sigma_max + ) + + # Generate video + sample = model.generate_samples_from_batch( + data_batch, + guidance=guidance, + state_shape=state_shape, + num_steps=num_steps, + is_negative_prompt=is_negative_prompt, + seed=seed, + x_sigma_max=x_sigma_max, + ) + + return sample + + +def generate_world_from_video( + model: DiffusionV2WModel, + state_shape: list[int], + is_negative_prompt: bool, + data_batch: dict, + guidance: float, + num_steps: int, + seed: int, + condition_latent: torch.Tensor, + num_input_frames: int, +) -> Tuple[np.array, list, list]: + """Generate video using a conditioning video/image input. + + Args: + model (DiffusionV2WModel): The diffusion model instance + state_shape (list[int]): Shape of the latent state [C,T,H,W] + is_negative_prompt (bool): Whether negative prompt is provided + data_batch (dict): Batch containing model inputs including text embeddings + guidance (float): Classifier-free guidance scale for sampling + num_steps (int): Number of diffusion sampling steps + seed (int): Random seed for generation + condition_latent (torch.Tensor): Latent tensor from conditioning video/image file + num_input_frames (int): Number of input frames + + Returns: + np.array: Generated video frames in shape [T,H,W,C], range [0,255] + """ + assert not model.config.conditioner.video_cond_bool.sample_tokens_start_from_p_or_i, "not supported" + augment_sigma = DEFAULT_AUGMENT_SIGMA + + if condition_latent.shape[2] < state_shape[1]: + # Padding condition latent to state shape + b, c, t, h, w = condition_latent.shape + condition_latent = torch.cat( + [ + condition_latent, + condition_latent.new_zeros(b, c, state_shape[1] - t, h, w), + ], + dim=2, + ).contiguous() + num_of_latent_condition = compute_num_latent_frames(model, num_input_frames) + x_sigma_max = ( + misc.arch_invariant_rand( + (1,) + tuple(state_shape), + torch.float32, + model.tensor_kwargs["device"], + seed, + ) + * model.sde.sigma_max + ) + + sample = model.generate_samples_from_batch( + data_batch, + guidance=guidance, + state_shape=state_shape, + num_steps=num_steps, + is_negative_prompt=is_negative_prompt, + seed=seed, + condition_latent=condition_latent, + num_condition_t=num_of_latent_condition, + condition_video_augment_sigma_in_inference=augment_sigma, + x_sigma_max=x_sigma_max, + ) + return sample + + +def read_video_or_image_into_frames_BCTHW( + input_path: str, + input_path_format: str = "mp4", + H: int = None, + W: int = None, + normalize: bool = True, + max_frames: int = -1, + also_return_fps: bool = False, +) -> torch.Tensor: + """Read video or image file and convert to tensor format. + + Args: + input_path (str): Path to input video/image file + input_path_format (str): Format of input file (default: "mp4") + H (int, optional): Height to resize frames to + W (int, optional): Width to resize frames to + normalize (bool): Whether to normalize pixel values to [-1,1] (default: True) + max_frames (int): Maximum number of frames to read (-1 for all frames) + also_return_fps (bool): Whether to return fps along with frames + + Returns: + torch.Tensor | tuple: Video tensor in shape [B,C,T,H,W], optionally with fps if requested + """ + log.debug(f"Reading video from {input_path}") + + loaded_data = load_from_fileobj(input_path, format=input_path_format) + frames, meta_data = loaded_data + if input_path.endswith(".png") or input_path.endswith(".jpg") or input_path.endswith(".jpeg"): + frames = np.array(frames[0]) # HWC, [0,255] + if frames.shape[-1] > 3: # RGBA, set the transparent to white + # Separate the RGB and Alpha channels + rgb_channels = frames[..., :3] + alpha_channel = frames[..., 3] / 255.0 # Normalize alpha channel to [0, 1] + + # Create a white background + white_bg = np.ones_like(rgb_channels) * 255 # White background in RGB + + # Blend the RGB channels with the white background based on the alpha channel + frames = (rgb_channels * alpha_channel[..., None] + white_bg * (1 - alpha_channel[..., None])).astype( + np.uint8 + ) + frames = [frames] + fps = 0 + else: + fps = int(meta_data.get("fps")) + if max_frames != -1: + frames = frames[:max_frames] + input_tensor = np.stack(frames, axis=0) + input_tensor = einops.rearrange(input_tensor, "t h w c -> t c h w") + if normalize: + input_tensor = input_tensor / 128.0 - 1.0 + input_tensor = torch.from_numpy(input_tensor).bfloat16() # TCHW + log.debug(f"Raw data shape: {input_tensor.shape}") + if H is not None and W is not None: + input_tensor = transforms_F.resize( + input_tensor, + size=(H, W), # type: ignore + interpolation=transforms_F.InterpolationMode.BICUBIC, + antialias=True, + ) + input_tensor = einops.rearrange(input_tensor, "(b t) c h w -> b c t h w", b=1) + if normalize: + input_tensor = input_tensor.to("cuda") + log.debug(f"Load shape {input_tensor.shape} value {input_tensor.min()}, {input_tensor.max()}") + if also_return_fps: + return input_tensor, fps + return input_tensor + + +def compute_num_latent_frames(model: DiffusionV2WModel, num_input_frames: int, downsample_factor=8) -> int: + """This function computes the number of latent frames given the number of input frames. + Args: + model (DiffusionV2WModel): video generation model + num_input_frames (int): number of input frames + downsample_factor (int): downsample factor for temporal reduce + Returns: + int: number of latent frames + """ + num_latent_frames = ( + num_input_frames + // model.tokenizer.video_vae.pixel_chunk_duration + * model.tokenizer.video_vae.latent_chunk_duration + ) + if num_input_frames % model.tokenizer.video_vae.latent_chunk_duration == 1: + num_latent_frames += 1 + elif num_input_frames % model.tokenizer.video_vae.latent_chunk_duration > 1: + assert (num_input_frames % model.tokenizer.video_vae.pixel_chunk_duration - 1) % downsample_factor == 0, ( + f"num_input_frames % model.tokenizer.video_vae.pixel_chunk_duration - 1 must be divisible by {downsample_factor}" + ) + num_latent_frames += ( + 1 + (num_input_frames % model.tokenizer.video_vae.pixel_chunk_duration - 1) // downsample_factor + ) + + return num_latent_frames + + +def create_condition_latent_from_input_frames( + model: DiffusionV2WModel, + input_frames: torch.Tensor, + num_frames_condition: int = 25, +): + """Create condition latent for video generation from input frames. + + Takes the last num_frames_condition frames from input as conditioning. + + Args: + model (DiffusionV2WModel): Video generation model + input_frames (torch.Tensor): Input video tensor [B,C,T,H,W], range [-1,1] + num_frames_condition (int): Number of frames to use for conditioning + + Returns: + tuple: (condition_latent, encode_input_frames) where: + - condition_latent (torch.Tensor): Encoded latent condition [B,C,T,H,W] + - encode_input_frames (torch.Tensor): Padded input frames used for encoding + """ + B, C, T, H, W = input_frames.shape + num_frames_encode = ( + model.tokenizer.pixel_chunk_duration + ) # (model.state_shape[1] - 1) / model.vae.pixel_chunk_duration + 1 + log.debug( + f"num_frames_encode not set, set it based on pixel chunk duration and model state shape: {num_frames_encode}" + ) + + log.debug( + f"Create condition latent from input frames {input_frames.shape}, value {input_frames.min()}, {input_frames.max()}, dtype {input_frames.dtype}" + ) + + assert input_frames.shape[2] >= num_frames_condition, ( + f"input_frames not enough for condition, require at least {num_frames_condition}, get {input_frames.shape[2]}, {input_frames.shape}" + ) + assert num_frames_encode >= num_frames_condition, ( + f"num_frames_encode should be larger than num_frames_condition, get {num_frames_encode}, {num_frames_condition}" + ) + + # Put the conditioal frames to the begining of the video, and pad the end with zero + condition_frames = input_frames[:, :, -num_frames_condition:] + padding_frames = condition_frames.new_zeros(B, C, num_frames_encode - num_frames_condition, H, W) + encode_input_frames = torch.cat([condition_frames, padding_frames], dim=2) + + log.debug( + f"create latent with input shape {encode_input_frames.shape} including padding {num_frames_encode - num_frames_condition} at the end" + ) + if hasattr(model, "n_cameras"): + encode_input_frames = einops.rearrange( + encode_input_frames, "(B V) C T H W -> B C (V T) H W", V=model.n_cameras + ) + latent = model.encode(encode_input_frames) + return latent, encode_input_frames + + +def get_condition_latent( + model: DiffusionV2WModel, + input_image_or_video_path: str, + num_input_frames: int = 1, + state_shape: list[int] = None, +): + """Get condition latent from input image/video file. + + Args: + model (DiffusionV2WModel): Video generation model + input_image_or_video_path (str): Path to conditioning image/video + num_input_frames (int): Number of input frames for video2world prediction + + Returns: + tuple: (condition_latent, input_frames) where: + - condition_latent (torch.Tensor): Encoded latent condition [B,C,T,H,W] + - input_frames (torch.Tensor): Input frames tensor [B,C,T,H,W] + """ + if state_shape is None: + state_shape = model.state_shape + assert num_input_frames > 0, "num_input_frames must be greater than 0" + + H, W = ( + state_shape[-2] * model.tokenizer.spatial_compression_factor, + state_shape[-1] * model.tokenizer.spatial_compression_factor, + ) + + input_path_format = input_image_or_video_path.split(".")[-1] + input_frames = read_video_or_image_into_frames_BCTHW( + input_image_or_video_path, + input_path_format=input_path_format, + H=H, + W=W, + ) + + condition_latent, _ = create_condition_latent_from_input_frames(model, input_frames, num_input_frames) + condition_latent = condition_latent.to(torch.bfloat16) + + return condition_latent + + +def get_condition_latent_multi_camera( + model: DiffusionMultiCameraV2WModel, + input_image_or_video_path: str, + num_input_frames: int = 1, + state_shape: list[int] = None, +): + """Get condition latent from input image/video file. This is the function for the multi-camera model where each camera has one latent condition frame. + + Args: + model (DiffusionMultiCameraV2WModel): Video generation model + input_image_or_video_path (str): Path to conditioning image/video + num_input_frames (int): Number of input frames for video2world prediction + + Returns: + tuple: (condition_latent, input_frames) where: + - condition_latent (torch.Tensor): Encoded latent condition [B,C,T,H,W] + - input_frames (torch.Tensor): Input frames tensor [B,C,T,H,W] + """ + if state_shape is None: + state_shape = model.state_shape + assert num_input_frames > 0, "num_input_frames must be greater than 0" + + H, W = ( + state_shape[-2] * model.tokenizer.spatial_compression_factor, + state_shape[-1] * model.tokenizer.spatial_compression_factor, + ) + input_path_format = input_image_or_video_path.split(".")[-1] + input_frames = read_video_or_image_into_frames_BCTHW( + input_image_or_video_path, + input_path_format=input_path_format, + H=H, + W=W, + ) + input_frames = einops.rearrange(input_frames, "B C (V T) H W -> (B V) C T H W", V=model.n_cameras) + condition_latent, _ = create_condition_latent_from_input_frames(model, input_frames, num_input_frames) + condition_latent = condition_latent.to(torch.bfloat16) + + return condition_latent, einops.rearrange(input_frames, "(B V) C T H W -> B C (V T) H W", V=model.n_cameras)[0] + + +def check_input_frames(input_path: str, required_frames: int) -> bool: + """Check if input video/image has sufficient frames. + + Args: + input_path: Path to input video or image + required_frames: Number of required frames + + Returns: + np.ndarray of frames if valid, None if invalid + """ + if input_path.endswith((".jpg", ".jpeg", ".png")): + if required_frames > 1: + log.error(f"Input ({input_path}) is an image but {required_frames} frames are required") + return False + return True # Let the pipeline handle image loading + # For video input + try: + vid = imageio.get_reader(input_path, "ffmpeg") + frame_count = vid.count_frames() + + if frame_count < required_frames: + log.error(f"Input video has {frame_count} frames but {required_frames} frames are required") + return False + else: + return True + except Exception as e: + log.error(f"Error reading video file {input_path}: {e}") + return False diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/text2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/text2world.py new file mode 100644 index 00000000..e2ba03fe --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/text2world.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import torch +from cosmos1.models.diffusion.inference.inference_utils import add_common_arguments, validate_args +from cosmos1.models.diffusion.inference.world_generation_pipeline import DiffusionText2WorldGenerationPipeline +from cosmos1.utils import log, misc +from cosmos1.utils.io import read_prompts_from_file, save_video + + +torch.enable_grad(False) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Text to world generation demo script") + # Add common arguments + add_common_arguments(parser) + + # Add text2world specific arguments + parser.add_argument( + "--diffusion_transformer_dir", + type=str, + default="Cosmos-1.0-Diffusion-7B-Text2World", + help="DiT model weights directory name relative to checkpoint_dir", + choices=[ + "Cosmos-1.0-Diffusion-7B-Text2World", + "Cosmos-1.0-Diffusion-14B-Text2World", + ], + ) + parser.add_argument( + "--prompt_upsampler_dir", + type=str, + default="Cosmos-1.0-Prompt-Upsampler-12B-Text2World", + help="Prompt upsampler weights directory relative to checkpoint_dir", + ) + + parser.add_argument( + "--word_limit_to_skip_upsampler", + type=int, + default=250, + help="Skip prompt upsampler for better robustness if the number of words in the prompt is greater than this value", + ) + + return parser.parse_args() + + +def demo(cfg): + """Run text-to-world generation demo. + + This function handles the main text-to-world generation pipeline, including: + - Setting up the random seed for reproducibility + - Initializing the generation pipeline with the provided configuration + - Processing single or multiple prompts from input + - Generating videos from text prompts + - Saving the generated videos and corresponding prompts to disk + + Args: + cfg (argparse.Namespace): Configuration namespace containing: + - Model configuration (checkpoint paths, model settings) + - Generation parameters (guidance, steps, dimensions) + - Input/output settings (prompts, save paths) + - Performance options (model offloading settings) + + The function will save: + - Generated MP4 video files + - Text files containing the processed prompts + + If guardrails block the generation, a critical log message is displayed + and the function continues to the next prompt if available. + """ + misc.set_random_seed(cfg.seed) + inference_type = "text2world" + validate_args(cfg, inference_type) + + # Initialize text2world generation model pipeline + pipeline = DiffusionText2WorldGenerationPipeline( + inference_type=inference_type, + checkpoint_dir=cfg.checkpoint_dir, + checkpoint_name=cfg.diffusion_transformer_dir, + prompt_upsampler_dir=cfg.prompt_upsampler_dir, + enable_prompt_upsampler=not cfg.disable_prompt_upsampler, + offload_network=cfg.offload_diffusion_transformer, + offload_tokenizer=cfg.offload_tokenizer, + offload_text_encoder_model=cfg.offload_text_encoder_model, + offload_prompt_upsampler=cfg.offload_prompt_upsampler, + offload_guardrail_models=cfg.offload_guardrail_models, + guidance=cfg.guidance, + num_steps=cfg.num_steps, + height=cfg.height, + width=cfg.width, + fps=cfg.fps, + num_video_frames=cfg.num_video_frames, + seed=cfg.seed, + ) + + # Handle multiple prompts if prompt file is provided + if cfg.batch_input_path: + log.info(f"Reading batch inputs from path: {args.batch_input_path}") + prompts = read_prompts_from_file(cfg.batch_input_path) + else: + # Single prompt case + prompts = [{"prompt": cfg.prompt}] + + os.makedirs(cfg.video_save_folder, exist_ok=True) + for i, input_dict in enumerate(prompts): + current_prompt = input_dict.get("prompt", None) + if current_prompt is None: + log.critical("Prompt is missing, skipping world generation.") + continue + + # Generate video + generated_output = pipeline.generate(current_prompt, cfg.negative_prompt, cfg.word_limit_to_skip_upsampler) + if generated_output is None: + log.critical("Guardrail blocked text2world generation.") + continue + video, prompt = generated_output + + if cfg.batch_input_path: + video_save_path = os.path.join(cfg.video_save_folder, f"{i}.mp4") + prompt_save_path = os.path.join(cfg.video_save_folder, f"{i}.txt") + else: + video_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.mp4") + prompt_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.txt") + + # Save video + save_video( + video=video, + fps=cfg.fps, + H=cfg.height, + W=cfg.width, + video_save_quality=5, + video_save_path=video_save_path, + ) + + # Save prompt to text file alongside video + with open(prompt_save_path, "wb") as f: + f.write(prompt.encode("utf-8")) + + log.info(f"Saved video to {video_save_path}") + log.info(f"Saved prompt to {prompt_save_path}") + + +if __name__ == "__main__": + args = parse_arguments() + demo(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/text2world_sample_multi_view_driving.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/text2world_sample_multi_view_driving.py new file mode 100644 index 00000000..fe484dd8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/text2world_sample_multi_view_driving.py @@ -0,0 +1,201 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import torch +from cosmos1.models.diffusion.inference.inference_utils import add_common_arguments, remove_argument, validate_args +from cosmos1.models.diffusion.inference.world_generation_pipeline import DiffusionText2WorldMultiViewGenerationPipeline +from cosmos1.utils import log, misc +from cosmos1.utils.io import save_video + + +torch.enable_grad(False) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Text to world generation demo script") + # Add common arguments + add_common_arguments(parser) + remove_argument(parser, "width") + remove_argument(parser, "height") + remove_argument(parser, "num_video_frames") + parser.add_argument("--height", type=int, default=480, help="Height of video to sample") + parser.add_argument("--width", type=int, default=848, help="Width of video to sample") + + parser.add_argument( + "--num_video_frames", + type=int, + default=57, + choices=[57], + help="Number of video frames to sample, this is per-camera frame number.", + ) + # Add text2world specific arguments + parser.add_argument( + "--diffusion_transformer_dir", + type=str, + default="Cosmos-1.0-Diffusion-7B-Text2World", + help="DiT model weights directory name relative to checkpoint_dir", + choices=[ + "Cosmos-1.0-Diffusion-7B-Text2World", + "Cosmos-1.0-Diffusion-14B-Text2World", + "Cosmos-1.1-Diffusion-7B-Text2World-Sample-Driving-Multiview", + ], + ) + parser.add_argument( + "--prompt_left", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing to the left. ", + help="Text prompt for generating left camera view video", + ) + parser.add_argument( + "--prompt_right", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing to the right.", + help="Text prompt for generating right camera view video", + ) + + parser.add_argument( + "--prompt_back", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing backwards.", + help="Text prompt for generating rear camera view video", + ) + parser.add_argument( + "--prompt_back_left", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing the rear left side.", + help="Text prompt for generating left camera view video", + ) + + parser.add_argument( + "--prompt_back_right", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing the rear right side.", + help="Text prompt for generating right camera view video", + ) + parser.add_argument( + "--frame_repeat_negative_condition", + type=float, + default=10.0, + help="frame_repeat number to be used as negative condition", + ) + + return parser.parse_args() + + +def demo(cfg): + """Run multi-view text-to-world generation demo. + + This function handles the main text-to-world generation pipeline, including: + - Setting up the random seed for reproducibility + - Initializing the generation pipeline with the provided configuration + - Processing single or multiple prompts from input + - Generating videos from text prompts + - Saving the generated videos and corresponding prompts to disk + + Args: + cfg (argparse.Namespace): Configuration namespace containing: + - Model configuration (checkpoint paths, model settings) + - Generation parameters (guidance, steps, dimensions) + - Input/output settings (prompts, save paths) + - Performance options (model offloading settings) + + The function will save: + - Generated MP4 video files + - Text files containing the processed prompts + + If guardrails block the generation, a critical log message is displayed + and the function continues to the next prompt if available. + """ + misc.set_random_seed(cfg.seed) + inference_type = "text2world" + validate_args(cfg, inference_type) + + # Initialize text2world generation model pipeline + pipeline = DiffusionText2WorldMultiViewGenerationPipeline( + inference_type=inference_type, + checkpoint_dir=cfg.checkpoint_dir, + checkpoint_name=cfg.diffusion_transformer_dir, + offload_network=cfg.offload_diffusion_transformer, + offload_tokenizer=cfg.offload_tokenizer, + offload_text_encoder_model=cfg.offload_text_encoder_model, + guidance=cfg.guidance, + num_steps=cfg.num_steps, + height=cfg.height, + width=cfg.width, + fps=cfg.fps, + num_video_frames=cfg.num_video_frames, + frame_repeat_negative_condition=cfg.frame_repeat_negative_condition, + seed=cfg.seed, + ) + + prompts = { + "prompt": cfg.prompt, + "prompt_left": cfg.prompt_left, + "prompt_right": cfg.prompt_right, + "prompt_back": cfg.prompt_back, + "prompt_back_left": cfg.prompt_back_left, + "prompt_back_right": cfg.prompt_back_right, + } + + os.makedirs(cfg.video_save_folder, exist_ok=True) + + # Generate video + generated_output = pipeline.generate(prompts) + if generated_output is None: + log.critical("Guardrail blocked text2world generation.") + return + [video_grid, video], prompt = generated_output + + video_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.mp4") + video_grid_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}_grid.mp4") + + prompt_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.txt") + + # Save video + save_video( + video=video, + fps=cfg.fps, + H=cfg.height, + W=cfg.width, + video_save_quality=10, + video_save_path=video_save_path, + ) + + save_video( + video=video_grid, + fps=cfg.fps, + H=cfg.height * 2, + W=cfg.width * 3, + video_save_quality=5, + video_save_path=video_grid_save_path, + ) + + # Save prompt to text file alongside video + with open(prompt_save_path, "wb") as f: + for key, value in prompt.items(): + f.write(value.encode("utf-8")) + f.write("\n".encode("utf-8")) + + log.info(f"Saved video to {video_save_path}") + log.info(f"Saved prompt to {prompt_save_path}") + + +if __name__ == "__main__": + args = parse_arguments() + demo(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/video2world.py new file mode 100644 index 00000000..43d1eedd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/video2world.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import torch +from cosmos1.models.diffusion.inference.inference_utils import add_common_arguments, check_input_frames, validate_args +from cosmos1.models.diffusion.inference.world_generation_pipeline import DiffusionVideo2WorldGenerationPipeline +from cosmos1.utils import log, misc +from cosmos1.utils.io import read_prompts_from_file, save_video + + +torch.enable_grad(False) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Video to world generation demo script") + # Add common arguments + add_common_arguments(parser) + + # Add video2world specific arguments + parser.add_argument( + "--diffusion_transformer_dir", + type=str, + default="Cosmos-1.0-Diffusion-7B-Video2World", + help="DiT model weights directory name relative to checkpoint_dir", + choices=[ + "Cosmos-1.0-Diffusion-7B-Video2World", + "Cosmos-1.0-Diffusion-14B-Video2World", + ], + ) + parser.add_argument( + "--prompt_upsampler_dir", + type=str, + default="Pixtral-12B", + help="Prompt upsampler weights directory relative to checkpoint_dir", + ) + parser.add_argument( + "--input_image_or_video_path", + type=str, + help="Input video/image path for generating a single video", + ) + parser.add_argument( + "--num_input_frames", + type=int, + default=1, + help="Number of input frames for video2world prediction", + choices=[1, 9], + ) + + return parser.parse_args() + + +def demo(cfg): + """Run video-to-world generation demo. + + This function handles the main video-to-world generation pipeline, including: + - Setting up the random seed for reproducibility + - Initializing the generation pipeline with the provided configuration + - Processing single or multiple prompts/images/videos from input + - Generating videos from prompts and images/videos + - Saving the generated videos and corresponding prompts to disk + + Args: + cfg (argparse.Namespace): Configuration namespace containing: + - Model configuration (checkpoint paths, model settings) + - Generation parameters (guidance, steps, dimensions) + - Input/output settings (prompts/images/videos, save paths) + - Performance options (model offloading settings) + + The function will save: + - Generated MP4 video files + - Text files containing the processed prompts + + If guardrails block the generation, a critical log message is displayed + and the function continues to the next prompt if available. + """ + misc.set_random_seed(cfg.seed) + inference_type = "video2world" + validate_args(cfg, inference_type) + + # Initialize video2world generation model pipeline + pipeline = DiffusionVideo2WorldGenerationPipeline( + inference_type=inference_type, + checkpoint_dir=cfg.checkpoint_dir, + checkpoint_name=cfg.diffusion_transformer_dir, + prompt_upsampler_dir=cfg.prompt_upsampler_dir, + enable_prompt_upsampler=not cfg.disable_prompt_upsampler, + offload_network=cfg.offload_diffusion_transformer, + offload_tokenizer=cfg.offload_tokenizer, + offload_text_encoder_model=cfg.offload_text_encoder_model, + offload_prompt_upsampler=cfg.offload_prompt_upsampler, + offload_guardrail_models=cfg.offload_guardrail_models, + guidance=cfg.guidance, + num_steps=cfg.num_steps, + height=cfg.height, + width=cfg.width, + fps=cfg.fps, + num_video_frames=cfg.num_video_frames, + seed=cfg.seed, + num_input_frames=cfg.num_input_frames, + ) + + # Handle multiple prompts if prompt file is provided + if cfg.batch_input_path: + log.info(f"Reading batch inputs from path: {args.batch_input_path}") + prompts = read_prompts_from_file(cfg.batch_input_path) + else: + # Single prompt case + prompts = [{"prompt": cfg.prompt, "visual_input": cfg.input_image_or_video_path}] + + os.makedirs(cfg.video_save_folder, exist_ok=True) + for i, input_dict in enumerate(prompts): + current_prompt = input_dict.get("prompt", None) + if current_prompt is None and cfg.disable_prompt_upsampler: + log.critical("Prompt is missing, skipping world generation.") + continue + current_image_or_video_path = input_dict.get("visual_input", None) + if current_image_or_video_path is None: + log.critical("Visual input is missing, skipping world generation.") + continue + + # Check input frames + if not check_input_frames(current_image_or_video_path, cfg.num_input_frames): + continue + + # Generate video + generated_output = pipeline.generate( + prompt=current_prompt, + image_or_video_path=current_image_or_video_path, + negative_prompt=cfg.negative_prompt, + ) + if generated_output is None: + log.critical("Guardrail blocked video2world generation.") + continue + video, prompt = generated_output + + if cfg.batch_input_path: + video_save_path = os.path.join(cfg.video_save_folder, f"{i}.mp4") + prompt_save_path = os.path.join(cfg.video_save_folder, f"{i}.txt") + else: + video_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.mp4") + prompt_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.txt") + + # Save video + save_video( + video=video, + fps=cfg.fps, + H=cfg.height, + W=cfg.width, + video_save_quality=5, + video_save_path=video_save_path, + ) + + # Save prompt to text file alongside video + with open(prompt_save_path, "wb") as f: + f.write(prompt.encode("utf-8")) + + log.info(f"Saved video to {video_save_path}") + log.info(f"Saved prompt to {prompt_save_path}") + + +if __name__ == "__main__": + args = parse_arguments() + demo(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/video2world_sample_multi_view_driving.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/video2world_sample_multi_view_driving.py new file mode 100644 index 00000000..be9b32a1 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/video2world_sample_multi_view_driving.py @@ -0,0 +1,214 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import torch +from cosmos1.models.diffusion.inference.inference_utils import add_common_arguments, remove_argument, validate_args +from cosmos1.models.diffusion.inference.world_generation_pipeline import ( + DiffusionVideo2WorldMultiViewGenerationPipeline, +) +from cosmos1.utils import log, misc +from cosmos1.utils.io import save_video + + +torch.enable_grad(False) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Video to world generation demo script") + # Add common arguments + add_common_arguments(parser) + remove_argument(parser, "width") + remove_argument(parser, "height") + remove_argument(parser, "num_video_frames") + parser.add_argument("--height", type=int, default=480, help="Height of video to sample") + parser.add_argument("--width", type=int, default=848, help="Width of video to sample") + + parser.add_argument( + "--num_video_frames", + type=int, + default=57, + choices=[57], + help="Number of video frames to sample, this is per-camera frame number.", + ) + # Add video2world specific arguments + parser.add_argument( + "--diffusion_transformer_dir", + type=str, + default="Cosmos-1.0-Diffusion-7B-Video2World", + help="DiT model weights directory name relative to checkpoint_dir", + choices=[ + "Cosmos-1.0-Diffusion-7B-Video2World", + "Cosmos-1.0-Diffusion-14B-Video2World", + "Cosmos-1.1-Diffusion-7B-Video2World-Sample-Driving-Multiview", + ], + ) + parser.add_argument( + "--prompt_left", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing to the left. ", + help="Text prompt for generating left camera view video", + ) + parser.add_argument( + "--prompt_right", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing to the right.", + help="Text prompt for generating right camera view video", + ) + + parser.add_argument( + "--prompt_back", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing backwards.", + help="Text prompt for generating rear camera view video", + ) + parser.add_argument( + "--prompt_back_left", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing the rear left side.", + help="Text prompt for generating left camera view video", + ) + + parser.add_argument( + "--prompt_back_right", + type=str, + default="The video is captured from a camera mounted on a car. The camera is facing the rear right side.", + help="Text prompt for generating right camera view video", + ) + parser.add_argument( + "--frame_repeat_negative_condition", + type=float, + default=10.0, + help="frame_repeat number to be used as negative condition", + ) + parser.add_argument( + "--input_image_or_video_path", + type=str, + help="Input video/image path for generating a single video", + ) + parser.add_argument( + "--num_input_frames", + type=int, + default=1, + help="Number of input frames for video2world prediction", + choices=[1, 9], + ) + + return parser.parse_args() + + +def demo(cfg): + """Run multi-view video-to-world generation demo. + + This function handles the main video-to-world generation pipeline, including: + - Setting up the random seed for reproducibility + - Initializing the generation pipeline with the provided configuration + - Processing single or multiple prompts/images/videos from input + - Generating videos from prompts and images/videos + - Saving the generated videos and corresponding prompts to disk + + Args: + cfg (argparse.Namespace): Configuration namespace containing: + - Model configuration (checkpoint paths, model settings) + - Generation parameters (guidance, steps, dimensions) + - Input/output settings (prompts/images/videos, save paths) + - Performance options (model offloading settings) + + The function will save: + - Generated MP4 video files + - Text files containing the processed prompts + + If guardrails block the generation, a critical log message is displayed + and the function continues to the next prompt if available. + """ + misc.set_random_seed(cfg.seed) + inference_type = "video2world" + validate_args(cfg, inference_type) + + # Initialize video2world generation model pipeline + pipeline = DiffusionVideo2WorldMultiViewGenerationPipeline( + inference_type=inference_type, + checkpoint_dir=cfg.checkpoint_dir, + checkpoint_name=cfg.diffusion_transformer_dir, + offload_network=cfg.offload_diffusion_transformer, + offload_tokenizer=cfg.offload_tokenizer, + offload_text_encoder_model=cfg.offload_text_encoder_model, + guidance=cfg.guidance, + num_steps=cfg.num_steps, + height=cfg.height, + width=cfg.width, + fps=cfg.fps, + num_video_frames=cfg.num_video_frames, + frame_repeat_negative_condition=cfg.frame_repeat_negative_condition, + seed=cfg.seed, + num_input_frames=cfg.num_input_frames, + ) + + prompts = { + "prompt": cfg.prompt, + "prompt_left": cfg.prompt_left, + "prompt_right": cfg.prompt_right, + "prompt_back": cfg.prompt_back, + "prompt_back_left": cfg.prompt_back_left, + "prompt_back_right": cfg.prompt_back_right, + } + + os.makedirs(cfg.video_save_folder, exist_ok=True) + generated_output = pipeline.generate( + prompts, + image_or_video_path=cfg.input_image_or_video_path, + ) + + if generated_output is None: + log.critical("Guardrail blocked video2world generation.") + return + [video_grid, video], prompt = generated_output + + video_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.mp4") + video_grid_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}_grid.mp4") + prompt_save_path = os.path.join(cfg.video_save_folder, f"{cfg.video_save_name}.txt") + # Save video + save_video( + video=video, + fps=cfg.fps, + H=cfg.height, + W=cfg.width, + video_save_quality=10, + video_save_path=video_save_path, + ) + save_video( + video=video_grid, + fps=cfg.fps, + H=cfg.height * 2, + W=cfg.width * 3, + video_save_quality=5, + video_save_path=video_grid_save_path, + ) + # Save prompt to text file alongside video + with open(prompt_save_path, "wb") as f: + for key, value in prompt.items(): + f.write(value.encode("utf-8")) + f.write("\n".encode("utf-8")) + + log.info(f"Saved video to {video_save_path}") + log.info(f"Saved prompt to {prompt_save_path}") + + +if __name__ == "__main__": + args = parse_arguments() + demo(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/world_generation_pipeline.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/world_generation_pipeline.py new file mode 100644 index 00000000..12fb746d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/inference/world_generation_pipeline.py @@ -0,0 +1,1072 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import os +from typing import Any, Optional + +import einops +import numpy as np +import torch +from cosmos1.models.common.base_world_generation_pipeline import BaseWorldGenerationPipeline +from cosmos1.models.diffusion.inference.inference_utils import ( + generate_world_from_text, + generate_world_from_video, + get_condition_latent, + get_condition_latent_multi_camera, + get_video_batch, + get_video_batch_for_multi_camera_model, + load_model_by_config, + load_network_model, + load_tokenizer_model, +) +from cosmos1.models.diffusion.model.model_sample_multiview_driving import ( + DiffusionMultiCameraT2WModel, + DiffusionMultiCameraV2WModel, +) +from cosmos1.models.diffusion.model.model_t2w import DiffusionT2WModel +from cosmos1.models.diffusion.model.model_v2w import DiffusionV2WModel +from cosmos1.models.diffusion.prompt_upsampler.text2world_prompt_upsampler_inference import ( + create_prompt_upsampler, + run_chat_completion, +) +from cosmos1.models.diffusion.prompt_upsampler.video2world_prompt_upsampler_inference import ( + create_vlm_prompt_upsampler, + prepare_dialog, +) +from cosmos1.models.diffusion.prompt_upsampler.video2world_prompt_upsampler_inference import ( + run_chat_completion as run_chat_completion_vlm, +) +from cosmos1.utils import log + + +MODEL_NAME_DICT = { + "Cosmos-1.0-Diffusion-7B-Text2World": "Cosmos_1_0_Diffusion_Text2World_7B", + "Cosmos-1.0-Diffusion-14B-Text2World": "Cosmos_1_0_Diffusion_Text2World_14B", + "Cosmos-1.0-Diffusion-7B-Video2World": "Cosmos_1_0_Diffusion_Video2World_7B", + "Cosmos-1.0-Diffusion-14B-Video2World": "Cosmos_1_0_Diffusion_Video2World_14B", + "Cosmos-1.1-Diffusion-7B-Text2World-Sample-Driving-Multiview": "Cosmos_1_1_Diffusion_Multi_Camera_Text2World_7B", + "Cosmos-1.1-Diffusion-7B-Video2World-Sample-Driving-Multiview": "Cosmos_1_1_Diffusion_Multi_Camera_Video2World_7B", +} + + +class DiffusionText2WorldGenerationPipeline(BaseWorldGenerationPipeline): + def __init__( + self, + inference_type: str, + checkpoint_dir: str, + checkpoint_name: str, + prompt_upsampler_dir: Optional[str] = None, + enable_prompt_upsampler: bool = True, + has_text_input: bool = True, + offload_network: bool = False, + offload_tokenizer: bool = False, + offload_text_encoder_model: bool = False, + offload_prompt_upsampler: bool = False, + offload_guardrail_models: bool = False, + guidance: float = 7.0, + num_steps: int = 35, + height: int = 704, + width: int = 1280, + fps: int = 24, + num_video_frames: int = 121, + seed: int = 0, + ): + """Initialize the diffusion world generation pipeline. + + Args: + inference_type: Type of world generation ('text2world' or 'video2world') + checkpoint_dir: Base directory containing model checkpoints + checkpoint_name: Name of the diffusion transformer checkpoint to use + prompt_upsampler_dir: Directory containing prompt upsampler model weights + enable_prompt_upsampler: Whether to use prompt upsampling + has_text_input: Whether the pipeline takes text input for world generation + offload_network: Whether to offload diffusion transformer after inference + offload_tokenizer: Whether to offload tokenizer after inference + offload_text_encoder_model: Whether to offload T5 model after inference + offload_prompt_upsampler: Whether to offload prompt upsampler + offload_guardrail_models: Whether to offload guardrail models + guidance: Classifier-free guidance scale + num_steps: Number of diffusion sampling steps + height: Height of output video + width: Width of output video + fps: Frames per second of output video + num_video_frames: Number of frames to generate + seed: Random seed for sampling + """ + assert inference_type in [ + "text2world", + "video2world", + ], "Invalid inference_type, must be 'text2world' or 'video2world'" + + self.model_name = MODEL_NAME_DICT[checkpoint_name] + self.guidance = guidance + self.num_steps = num_steps + self.height = height + self.width = width + self.fps = fps + self.num_video_frames = num_video_frames + self.seed = seed + + super().__init__( + inference_type=inference_type, + checkpoint_dir=checkpoint_dir, + checkpoint_name=checkpoint_name, + has_text_input=has_text_input, + offload_network=offload_network, + offload_tokenizer=offload_tokenizer, + offload_text_encoder_model=offload_text_encoder_model, + offload_guardrail_models=offload_guardrail_models, + ) + self.prompt_upsampler_dir = prompt_upsampler_dir + self.enable_prompt_upsampler = enable_prompt_upsampler + self.offload_prompt_upsampler = offload_prompt_upsampler + + self.prompt_upsampler = None + if enable_prompt_upsampler and not offload_prompt_upsampler: + self._load_prompt_upsampler_model() + + def _load_prompt_upsampler_model(self): + self.prompt_upsampler = create_prompt_upsampler( + checkpoint_dir=os.path.join(self.checkpoint_dir, self.prompt_upsampler_dir), + ) + + def _load_model(self): + self.model = load_model_by_config( + config_job_name=self.model_name, + config_file="cosmos1/models/diffusion/config/config.py", + model_class=DiffusionT2WModel, + ) + + def _load_network(self): + load_network_model(self.model, f"{self.checkpoint_dir}/{self.checkpoint_name}/model.pt") + + def _load_tokenizer(self): + load_tokenizer_model(self.model, f"{self.checkpoint_dir}/Cosmos-1.0-Tokenizer-CV8x8x8") + + def _offload_prompt_upsampler_model(self): + """Move prompt enhancement model to CPU/disk. + + Offloads prompt upsampling model after processing input + to reduce GPU memory usage. + """ + if self.prompt_upsampler: + del self.prompt_upsampler + self.prompt_upsampler = None + gc.collect() + torch.cuda.empty_cache() + + def _run_prompt_upsampler_on_prompt(self, prompt: str) -> str: + """Enhance the input prompt using the prompt upsampler model. + + Args: + prompt: Raw text prompt to be enhanced + + Returns: + str: Enhanced version of the input prompt with more descriptive details + """ + upsampled_prompt = run_chat_completion(self.prompt_upsampler, prompt) + log.info(f"Upsampled prompt: {upsampled_prompt}") + return upsampled_prompt + + def _run_prompt_upsampler_on_prompt_with_offload(self, *args: Any, **kwargs: Any) -> str: + """Enhance prompt with prompt upsampler model. + + Args: + *args: Positional arguments + **kwargs: Keyword arguments + + Returns: + Enhanced prompt string + """ + if self.offload_prompt_upsampler: + self._load_prompt_upsampler_model() + + enhanced_prompt = self._run_prompt_upsampler_on_prompt(*args, **kwargs) + + if self.offload_prompt_upsampler: + self._offload_prompt_upsampler_model() + + return enhanced_prompt + + def _run_tokenizer_decoding(self, sample: torch.Tensor) -> np.ndarray: + """Decode latent samples to video frames using the tokenizer decoder. + + Args: + sample: Latent tensor from diffusion model [B, C, T, H, W] + + Returns: + np.ndarray: Decoded video frames as uint8 numpy array [T, H, W, C] + with values in range [0, 255] + """ + # Decode video + video = (1.0 + self.model.decode(sample)).clamp(0, 2) / 2 # [B, 3, T, H, W] + video = (video[0].permute(1, 2, 3, 0) * 255).to(torch.uint8).cpu().numpy() + + return video + + def _run_model( + self, + embedding: torch.Tensor, + negative_prompt_embedding: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Generate video latents using the diffusion model. + + Args: + embedding: Text embedding tensor from text encoder + negative_prompt_embedding: Optional embedding for negative prompt guidance + + Returns: + torch.Tensor: Generated video latents before tokenizer decoding + + Note: + The model and tokenizer are automatically offloaded after inference + if offloading is enabled in the config. + """ + # Get video batch and state shape + data_batch, state_shape = get_video_batch( + model=self.model, + prompt_embedding=embedding, + negative_prompt_embedding=negative_prompt_embedding, + height=self.height, + width=self.width, + fps=self.fps, + num_video_frames=self.num_video_frames, + ) + + # Generate video frames + sample = generate_world_from_text( + model=self.model, + state_shape=state_shape, + is_negative_prompt=True if negative_prompt_embedding is not None else False, + data_batch=data_batch, + guidance=self.guidance, + num_steps=self.num_steps, + seed=self.seed, + ) + + return sample + + def _run_model_with_offload( + self, prompt_embedding: torch.Tensor, negative_prompt_embedding: Optional[torch.Tensor] = None + ) -> np.ndarray: + """Generate world representation with automatic model offloading. + + Wraps the core generation process with model loading/offloading logic + to minimize GPU memory usage during inference. + + Args: + *args: Positional arguments passed to _run_model + **kwargs: Keyword arguments passed to _run_model + + Returns: + np.ndarray: Generated world representation as numpy array + """ + if self.offload_network: + self._load_network() + + if self.offload_tokenizer: + self._load_tokenizer() + + sample = self._run_model(prompt_embedding, negative_prompt_embedding) + + if self.offload_network: + self._offload_network() + + if self.offload_tokenizer: + self._load_tokenizer() + + sample = self._run_tokenizer_decoding(sample) + + if self.offload_tokenizer: + self._offload_tokenizer() + return sample + + def generate( + self, + prompt: str, + negative_prompt: Optional[str] = None, + word_limit_to_skip_upsampler: Optional[int] = None, + ) -> tuple[np.ndarray, str] | None: + """Generate video from text prompt with optional negative prompt guidance. + + Pipeline steps: + 1. Run safety checks on input prompt + 2. Enhance prompt using upsampler if enabled + 3. Run safety checks on upsampled prompt if applicable + 4. Convert prompt to embeddings + 5. Generate video frames using diffusion + 6. Run safety checks and apply face blur on generated video frames + + Args: + prompt: Text description of desired video + negative_prompt: Optional text to guide what not to generate + word_limit_to_skip_upsampler: Skip prompt upsampler for better robustness if the number of words in the prompt is greater than this value + Returns: + tuple: ( + Generated video frames as uint8 np.ndarray [T, H, W, C], + Final prompt used for generation (may be enhanced) + ), or None if content fails guardrail safety checks + """ + log.info(f"Run with prompt: {prompt}") + log.info(f"Run with negative prompt: {negative_prompt}") + log.info(f"Run with prompt upsampler: {self.enable_prompt_upsampler}") + + log.info("Run guardrail on prompt") + is_safe = self._run_guardrail_on_prompt_with_offload(prompt) + if not is_safe: + log.critical("Input text prompt is not safe") + return None + log.info("Pass guardrail on prompt") + + # Enhance prompt + if self.enable_prompt_upsampler: + word_count = len(prompt.split()) + if word_limit_to_skip_upsampler is None or word_count <= word_limit_to_skip_upsampler: + log.info("Run prompt upsampler on prompt") + prompt = self._run_prompt_upsampler_on_prompt_with_offload(prompt) + log.info("Run guardrail on upsampled prompt") + is_safe = self._run_guardrail_on_prompt_with_offload(prompt=prompt) + if not is_safe: + log.critical("Upsampled text prompt is not safe") + return None + log.info("Pass guardrail on upsampled prompt") + else: + log.info( + f"Skip prompt upsampler for better robustness because the number of words ({word_count}) in the prompt is greater than {word_limit_to_skip_upsampler}" + ) + + log.info("Run text embedding on prompt") + if negative_prompt: + prompts = [prompt, negative_prompt] + else: + prompts = [prompt] + prompt_embeddings, _ = self._run_text_embedding_on_prompt_with_offload(prompts) + prompt_embedding = prompt_embeddings[0] + negative_prompt_embedding = prompt_embeddings[1] if negative_prompt else None + log.info("Finish text embedding on prompt") + + # Generate video + log.info("Run generation") + video = self._run_model_with_offload( + prompt_embedding, + negative_prompt_embedding=negative_prompt_embedding, + ) + log.info("Finish generation") + + log.info("Run guardrail on generated video") + video = self._run_guardrail_on_video_with_offload(video) + if video is None: + log.critical("Generated video is not safe") + return None + log.info("Pass guardrail on generated video") + + return video, prompt + + +class DiffusionVideo2WorldGenerationPipeline(DiffusionText2WorldGenerationPipeline): + def __init__( + self, + inference_type: str, + checkpoint_dir: str, + checkpoint_name: str, + prompt_upsampler_dir: Optional[str] = None, + enable_prompt_upsampler: bool = True, + has_text_input: bool = True, + offload_network: bool = False, + offload_tokenizer: bool = False, + offload_text_encoder_model: bool = False, + offload_prompt_upsampler: bool = False, + offload_guardrail_models: bool = False, + guidance: float = 7.0, + num_steps: int = 35, + height: int = 704, + width: int = 1280, + fps: int = 24, + num_video_frames: int = 121, + seed: int = 0, + num_input_frames: int = 1, + ): + """Initialize diffusion world generation pipeline. + + Args: + inference_type: Type of world generation ('text2world' or 'video2world') + checkpoint_dir: Base directory containing model checkpoints + checkpoint_name: Name of the diffusion transformer checkpoint to use + prompt_upsampler_dir: Directory containing prompt upsampler model weights + enable_prompt_upsampler: Whether to use prompt upsampling + has_text_input: Whether the pipeline takes text input for world generation + offload_network: Whether to offload diffusion transformer after inference + offload_tokenizer: Whether to offload tokenizer after inference + offload_text_encoder_model: Whether to offload T5 model after inference + offload_prompt_upsampler: Whether to offload prompt upsampler + offload_guardrail_models: Whether to offload guardrail models + guidance: Classifier-free guidance scale + num_steps: Number of diffusion sampling steps + height: Height of output video + width: Width of output video + fps: Frames per second of output video + num_video_frames: Number of frames to generate + seed: Random seed for sampling + num_input_frames: Number of latent conditions + """ + self.num_input_frames = num_input_frames + super().__init__( + inference_type=inference_type, + checkpoint_dir=checkpoint_dir, + checkpoint_name=checkpoint_name, + prompt_upsampler_dir=prompt_upsampler_dir, + enable_prompt_upsampler=enable_prompt_upsampler, + has_text_input=has_text_input, + offload_network=offload_network, + offload_tokenizer=offload_tokenizer, + offload_text_encoder_model=offload_text_encoder_model, + offload_prompt_upsampler=offload_prompt_upsampler, + offload_guardrail_models=offload_guardrail_models, + guidance=guidance, + num_steps=num_steps, + height=height, + width=width, + fps=fps, + num_video_frames=num_video_frames, + seed=seed, + ) + + def _run_prompt_upsampler_on_prompt(self, image_or_video_path: str) -> str: + """Enhance the input prompt using visual context from the conditioning image. + + Args: + image_or_video_path: Path to conditioning image or video used for visual context + + Returns: + str: Enhanced prompt incorporating visual details from the image + """ + dialog = prepare_dialog(image_or_video_path) + upsampled_prompt = run_chat_completion_vlm( + self.prompt_upsampler, dialog, max_gen_len=400, temperature=0.01, top_p=0.9, logprobs=False + ) + log.info(f"Upsampled prompt: {upsampled_prompt}") + return upsampled_prompt + + def _load_prompt_upsampler_model(self): + self.prompt_upsampler = create_vlm_prompt_upsampler( + checkpoint_dir=os.path.join(self.checkpoint_dir, self.prompt_upsampler_dir), + ) + + def _load_model(self): + self.model = load_model_by_config( + config_job_name=self.model_name, + config_file="cosmos1/models/diffusion/config/config.py", + model_class=DiffusionV2WModel, + ) + + def _run_model( + self, + embedding: torch.Tensor, + condition_latent: torch.Tensor, + negative_prompt_embedding: torch.Tensor | None = None, + ) -> torch.Tensor: + """Generate video frames using the diffusion model. + + Args: + embedding: Text embedding tensor from T5 encoder + condition_latent: Latent tensor from conditioning image or video + negative_prompt_embedding: Optional embedding for negative prompt guidance + + Returns: + Tensor of generated video frames + + Note: + Model and tokenizer are automatically offloaded after inference + if offloading is enabled. + """ + # Get video batch and state shape + data_batch, state_shape = get_video_batch( + model=self.model, + prompt_embedding=embedding, + negative_prompt_embedding=negative_prompt_embedding, + height=self.height, + width=self.width, + fps=self.fps, + num_video_frames=self.num_video_frames, + ) + + # Generate video frames + video = generate_world_from_video( + model=self.model, + state_shape=self.model.state_shape, + is_negative_prompt=True, + data_batch=data_batch, + guidance=self.guidance, + num_steps=self.num_steps, + seed=self.seed, + condition_latent=condition_latent, + num_input_frames=self.num_input_frames, + ) + + return video + + def _run_tokenizer_encoding(self, image_or_video_path: str) -> torch.Tensor: + """ + Encode image to latent space + + Args: + image_or_video_path: Path to conditioning image + + Returns: + torch.Tensor: Latent tensor from tokenizer encoding + """ + condition_latent = get_condition_latent( + model=self.model, + input_image_or_video_path=image_or_video_path, + num_input_frames=self.num_input_frames, + state_shape=self.model.state_shape, + ) + + return condition_latent + + def _run_model_with_offload( + self, + prompt_embedding: torch.Tensor, + image_or_video_path: str, + negative_prompt_embedding: Optional[torch.Tensor] = None, + ) -> np.ndarray: + """Generate world representation with automatic model offloading. + + Wraps the core generation process with model loading/offloading logic + to minimize GPU memory usage during inference. + + Args: + prompt_embedding: Text embedding tensor from T5 encoder + image_or_video_path: Path to conditioning image or video + negative_prompt_embedding: Optional embedding for negative prompt guidance + + Returns: + np.ndarray: Generated world representation as numpy array + """ + if self.offload_tokenizer: + self._load_tokenizer() + + condition_latent = self._run_tokenizer_encoding(image_or_video_path) + + if self.offload_network: + self._load_network() + + sample = self._run_model(prompt_embedding, condition_latent, negative_prompt_embedding) + + if self.offload_network: + self._offload_network() + + sample = self._run_tokenizer_decoding(sample) + + if self.offload_tokenizer: + self._offload_tokenizer() + + return sample + + def generate( + self, + prompt: str, + image_or_video_path: str, + negative_prompt: Optional[str] = None, + ) -> tuple[np.ndarray, str] | None: + """Generate video from text prompt and optional image. + + Pipeline steps: + 1. Run safety checks on input prompt + 2. Enhance prompt using upsampler if enabled + 3. Run safety checks on upsampled prompt if applicable + 4. Convert prompt to embeddings + 5. Generate video frames using diffusion + 6. Run safety checks and apply face blur on generated video frames + + Args: + prompt: Text description of desired video + image_or_video_path: Path to conditioning image or video + negative_prompt: Optional text to guide what not to generate + + Returns: + tuple: ( + Generated video frames as uint8 np.ndarray [T, H, W, C], + Final prompt used for generation (may be enhanced) + ), or None if content fails guardrail safety checks + """ + log.info(f"Run with prompt: {prompt}") + log.info(f"Run with image or video path: {image_or_video_path}") + log.info(f"Run with negative prompt: {negative_prompt}") + log.info(f"Run with prompt upsampler: {self.enable_prompt_upsampler}") + + if not self.enable_prompt_upsampler: + log.info("Run guardrail on prompt") + is_safe = self._run_guardrail_on_prompt_with_offload(prompt) + if not is_safe: + log.critical("Input text prompt is not safe") + return None + log.info("Pass guardrail on prompt") + else: + log.info("Run prompt upsampler on image or video, input prompt is not used") + prompt = self._run_prompt_upsampler_on_prompt_with_offload(image_or_video_path=image_or_video_path) + log.info("Run guardrail on upsampled prompt") + is_safe = self._run_guardrail_on_prompt_with_offload(prompt) + if not is_safe: + log.critical("Upsampled text prompt is not safe") + return None + log.info("Pass guardrail on upsampled prompt") + + log.info("Run text embedding on prompt") + if negative_prompt: + prompts = [prompt, negative_prompt] + else: + prompts = [prompt] + prompt_embeddings, _ = self._run_text_embedding_on_prompt_with_offload(prompts) + prompt_embedding = prompt_embeddings[0] + negative_prompt_embedding = prompt_embeddings[1] if negative_prompt else None + log.info("Finish text embedding on prompt") + + # Generate video + log.info("Run generation") + video = self._run_model_with_offload( + prompt_embedding, + negative_prompt_embedding=negative_prompt_embedding, + image_or_video_path=image_or_video_path, + ) + log.info("Finish generation") + + log.info("Run guardrail on generated video") + video = self._run_guardrail_on_video_with_offload(video) + if video is None: + log.critical("Generated video is not safe") + return None + log.info("Pass guardrail on generated video") + + return video, prompt + + +class DiffusionText2WorldMultiViewGenerationPipeline(DiffusionText2WorldGenerationPipeline): + def __init__( + self, + inference_type: str, + checkpoint_dir: str, + checkpoint_name: str, + prompt_upsampler_dir: Optional[str] = None, + has_text_input: bool = True, + offload_network: bool = False, + offload_tokenizer: bool = False, + offload_text_encoder_model: bool = False, + offload_prompt_upsampler: bool = False, + offload_guardrail_models: bool = False, + guidance: float = 7.0, + num_steps: int = 35, + height: int = 704, + width: int = 1280, + fps: int = 24, + num_video_frames: int = 121, + n_cameras: int = 6, + frame_repeat_negative_condition: int = 10, + seed: int = 0, + ): + """Initialize the diffusion multi-view world generation pipeline. + + Args: + inference_type: Type of world generation ('text2world' or 'video2world') + checkpoint_dir: Base directory containing model checkpoints + checkpoint_name: Name of the diffusion transformer checkpoint to use + prompt_upsampler_dir: Directory containing prompt upsampler model weights + enable_prompt_upsampler: Whether to use prompt upsampling + has_text_input: Whether the pipeline takes text input for world generation + offload_network: Whether to offload diffusion transformer after inference + offload_tokenizer: Whether to offload tokenizer after inference + offload_text_encoder_model: Whether to offload T5 model after inference + offload_prompt_upsampler: Whether to offload prompt upsampler + offload_guardrail_models: Whether to offload guardrail models + guidance: Classifier-free guidance scale + num_steps: Number of diffusion sampling steps + height: Height of output video + width: Width of output video + fps: Frames per second of output video + num_video_frames: Number of frames to generate + n_cameras: Number of cameras + frame_repeat_negative_condition: Number of frames to repeat to be used as negative condition. + seed: Random seed for sampling + """ + assert inference_type in [ + "text2world", + "video2world", + ], "Invalid inference_type, must be 'text2world' or 'video2world'" + + self.n_cameras = n_cameras + self.frame_repeat_negative_condition = frame_repeat_negative_condition + super().__init__( + inference_type=inference_type, + checkpoint_dir=checkpoint_dir, + checkpoint_name=checkpoint_name, + prompt_upsampler_dir=prompt_upsampler_dir, + enable_prompt_upsampler=False, + has_text_input=has_text_input, + offload_network=offload_network, + offload_tokenizer=offload_tokenizer, + offload_text_encoder_model=offload_text_encoder_model, + offload_prompt_upsampler=offload_prompt_upsampler, + offload_guardrail_models=offload_guardrail_models, + guidance=guidance, + num_steps=num_steps, + height=height, + width=width, + fps=fps, + num_video_frames=num_video_frames, + seed=seed, + ) + + def _load_model(self): + self.model = load_model_by_config( + config_job_name=self.model_name, + config_file="cosmos1/models/diffusion/config/config.py", + model_class=DiffusionMultiCameraT2WModel, + ) + + def _run_tokenizer_decoding(self, sample: torch.Tensor) -> np.ndarray: + """Decode latent samples to video frames using the tokenizer decoder. + + Args: + sample: Latent tensor from diffusion model [B, C, T, H, W] + + Returns: + np.ndarray: Decoded video frames as uint8 numpy array [T, H, W, C] + with values in range [0, 255] + """ + # Decode video + video = (1.0 + self.model.decode(sample)).clamp(0, 2) / 2 # [B, 3, T, H, W] + video_segments = einops.rearrange(video, "b c (v t) h w -> b c v t h w", v=self.n_cameras) + grid_video = torch.stack( + [video_segments[:, :, i] for i in [1, 0, 2, 4, 3, 5]], + dim=2, + ) + grid_video = einops.rearrange(grid_video, "b c (h w) t h1 w1 -> b c t (h h1) (w w1)", h=2, w=3) + grid_video = (grid_video[0].permute(1, 2, 3, 0) * 255).to(torch.uint8).cpu().numpy() + video = (video[0].permute(1, 2, 3, 0) * 255).to(torch.uint8).cpu().numpy() + + return [grid_video, video] + + def _run_model( + self, + embedding: torch.Tensor, + negative_prompt_embedding: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Generate video latents using the diffusion model. + + Args: + embedding: Text embedding tensor from text encoder + negative_prompt_embedding: Optional embedding for negative prompt guidance + + Returns: + torch.Tensor: Generated video latents before tokenizer decoding + + Note: + The model and tokenizer are automatically offloaded after inference + if offloading is enabled in the config. + """ + # Get video batch and state shape + data_batch, state_shape = get_video_batch_for_multi_camera_model( + model=self.model, + prompt_embedding=embedding, + height=self.height, + width=self.width, + fps=self.fps, + num_video_frames=self.num_video_frames * len(embedding), # number of cameras + frame_repeat_negative_condition=self.frame_repeat_negative_condition, + ) + + # Generate video frames + sample = generate_world_from_text( + model=self.model, + state_shape=state_shape, + is_negative_prompt=False, + data_batch=data_batch, + guidance=self.guidance, + num_steps=self.num_steps, + seed=self.seed, + ) + + return sample + + def generate( + self, + prompt: dict, + ) -> tuple[np.ndarray, str] | None: + """Generate video from text prompt with optional negative prompt guidance. + + Pipeline steps: + 1. Convert prompt to embeddings + 2. Generate video frames using diffusion + + Args: + prompt: A dictionary of text description of desired video. + Returns: + tuple: ( + Generated video frames as uint8 np.ndarray [T, H, W, C], + Final prompt used for generation (may be enhanced) + ), or None if content fails guardrail safety checks + """ + log.info(f"Run with prompt: {prompt}") + + prompts = [ + prompt["prompt"], + prompt["prompt_left"], + prompt["prompt_right"], + prompt["prompt_back"], + prompt["prompt_back_left"], + prompt["prompt_back_right"], + ] + prompt_embeddings, _ = self._run_text_embedding_on_prompt_with_offload(prompts) + log.info("Finish text embedding on prompt") + + # Generate video + log.info("Run generation") + videos = self._run_model_with_offload( + prompt_embeddings, + ) + log.info("Finish generation") + + return videos, prompt + + +class DiffusionVideo2WorldMultiViewGenerationPipeline(DiffusionText2WorldMultiViewGenerationPipeline): + def __init__( + self, + inference_type: str, + checkpoint_dir: str, + checkpoint_name: str, + prompt_upsampler_dir: Optional[str] = None, + enable_prompt_upsampler: bool = True, + has_text_input: bool = True, + offload_network: bool = False, + offload_tokenizer: bool = False, + offload_text_encoder_model: bool = False, + offload_prompt_upsampler: bool = False, + offload_guardrail_models: bool = False, + guidance: float = 7.0, + num_steps: int = 35, + height: int = 704, + width: int = 1280, + fps: int = 24, + num_video_frames: int = 121, + seed: int = 0, + num_input_frames: int = 1, + n_cameras: int = 6, + frame_repeat_negative_condition: int = 10, + ): + """Initialize diffusion world multi-view generation pipeline. + + Args: + inference_type: Type of world generation ('text2world' or 'video2world') + checkpoint_dir: Base directory containing model checkpoints + checkpoint_name: Name of the diffusion transformer checkpoint to use + prompt_upsampler_dir: Directory containing prompt upsampler model weights + enable_prompt_upsampler: Whether to use prompt upsampling + has_text_input: Whether the pipeline takes text input for world generation + offload_network: Whether to offload diffusion transformer after inference + offload_tokenizer: Whether to offload tokenizer after inference + offload_text_encoder_model: Whether to offload T5 model after inference + offload_prompt_upsampler: Whether to offload prompt upsampler + offload_guardrail_models: Whether to offload guardrail models + guidance: Classifier-free guidance scale + num_steps: Number of diffusion sampling steps + height: Height of output video + width: Width of output video + fps: Frames per second of output video + num_video_frames: Number of frames to generate + seed: Random seed for sampling + num_input_frames: Number of latent conditions + """ + self.num_input_frames = num_input_frames + super().__init__( + inference_type=inference_type, + checkpoint_dir=checkpoint_dir, + checkpoint_name=checkpoint_name, + prompt_upsampler_dir=prompt_upsampler_dir, + has_text_input=has_text_input, + offload_network=offload_network, + offload_tokenizer=offload_tokenizer, + offload_text_encoder_model=offload_text_encoder_model, + offload_prompt_upsampler=offload_prompt_upsampler, + offload_guardrail_models=offload_guardrail_models, + guidance=guidance, + num_steps=num_steps, + height=height, + width=width, + fps=fps, + num_video_frames=num_video_frames, + seed=seed, + n_cameras=n_cameras, + frame_repeat_negative_condition=frame_repeat_negative_condition, + ) + + def _load_model(self): + self.model = load_model_by_config( + config_job_name=self.model_name, + config_file="cosmos1/models/diffusion/config/config.py", + model_class=DiffusionMultiCameraV2WModel, + ) + + def _run_model( + self, + embedding: torch.Tensor, + condition_latent: torch.Tensor, + negative_prompt_embedding: torch.Tensor | None = None, + data_batch: dict = None, + state_shape: list = None, + ) -> torch.Tensor: + """Generate video frames using the diffusion model. + + Args: + embedding: Text embedding tensor from T5 encoder + condition_latent: Latent tensor from conditioning image or video + negative_prompt_embedding: Optional embedding for negative prompt guidance + + Returns: + Tensor of generated video frames + + Note: + Model and tokenizer are automatically offloaded after inference + if offloading is enabled. + """ + # Generate video frames + video = generate_world_from_video( + model=self.model, + state_shape=state_shape, + is_negative_prompt=False, + data_batch=data_batch, + guidance=self.guidance, + num_steps=self.num_steps, + seed=self.seed, + condition_latent=condition_latent, + num_input_frames=self.num_input_frames, + ) + + return video + + def _run_tokenizer_encoding(self, image_or_video_path: str, state_shape: list) -> torch.Tensor: + """ + Encode image to latent space + + Args: + image_or_video_path: Path to conditioning image + + Returns: + torch.Tensor: Latent tensor from tokenizer encoding + """ + condition_latent, condition_frames = get_condition_latent_multi_camera( + model=self.model, + input_image_or_video_path=image_or_video_path, + num_input_frames=self.num_input_frames, + state_shape=state_shape, + ) + + return condition_latent, condition_frames + + def _run_model_with_offload( + self, + prompt_embedding: torch.Tensor, + image_or_video_path: str, + negative_prompt_embedding: Optional[torch.Tensor] = None, + ) -> np.ndarray: + """Generate world representation with automatic model offloading. + + Wraps the core generation process with model loading/offloading logic + to minimize GPU memory usage during inference. + + Args: + prompt_embedding: Text embedding tensor from T5 encoder + image_or_video_path: Path to conditioning image or video + negative_prompt_embedding: Optional embedding for negative prompt guidance + + Returns: + np.ndarray: Generated world representation as numpy array + """ + if self.offload_tokenizer: + self._load_tokenizer() + + data_batch, state_shape = get_video_batch_for_multi_camera_model( + model=self.model, + prompt_embedding=prompt_embedding, + height=self.height, + width=self.width, + fps=self.fps, + num_video_frames=self.num_video_frames * len(prompt_embedding), # number of cameras + frame_repeat_negative_condition=self.frame_repeat_negative_condition, + ) + + condition_latent, condition_frames = self._run_tokenizer_encoding(image_or_video_path, state_shape) + + if self.offload_network: + self._load_network() + + sample = self._run_model( + prompt_embedding, condition_latent, negative_prompt_embedding, data_batch, state_shape + ) + + if self.offload_network: + self._offload_network() + + sample = self._run_tokenizer_decoding(sample) + + if self.offload_tokenizer: + self._offload_tokenizer() + + return sample + + def generate( + self, + prompt: dict, + image_or_video_path: str, + ) -> tuple[np.ndarray, str] | None: + """Generate video from text prompt with optional negative prompt guidance. + + Pipeline steps: + 1. Convert prompt to embeddings + 2. Generate video frames using diffusion + + Args: + prompt: A dictionary of text description of desired video. + Returns: + tuple: ( + Generated video frames as uint8 np.ndarray [T, H, W, C], + Final prompt used for generation (may be enhanced) + ), or None if content fails guardrail safety checks + """ + log.info(f"Run with prompt: {prompt}") + + prompts = [ + prompt["prompt"], + prompt["prompt_left"], + prompt["prompt_right"], + prompt["prompt_back"], + prompt["prompt_back_left"], + prompt["prompt_back_right"], + ] + prompt_embeddings, _ = self._run_text_embedding_on_prompt_with_offload(prompts) + log.info("Finish text embedding on prompt") + + # Generate video + log.info("Run generation") + video = self._run_model_with_offload( + prompt_embeddings, + image_or_video_path=image_or_video_path, + ) + log.info("Finish generation") + + return video, prompt diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_sample_multiview_driving.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_sample_multiview_driving.py new file mode 100644 index 00000000..73810975 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_sample_multiview_driving.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Union + +import torch +from cosmos1.models.diffusion.conditioner import VideoExtendCondition +from cosmos1.models.diffusion.model.model_t2w import DiffusionT2WModel +from cosmos1.models.diffusion.model.model_v2w import DiffusionV2WModel +from cosmos1.utils import log +from einops import rearrange + + +class DiffusionMultiCameraT2WModel(DiffusionT2WModel): + def __init__(self, config): + super().__init__(config) + self.n_cameras = config.net.n_cameras + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + state = rearrange(state, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + encoded_state = self.tokenizer.encode(state) + encoded_state = rearrange(encoded_state, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) * self.sigma_data + return encoded_state + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + latent = rearrange(latent, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + decoded_state = self.tokenizer.decode(latent / self.sigma_data) + decoded_state = rearrange(decoded_state, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + return decoded_state + + +class DiffusionMultiCameraV2WModel(DiffusionV2WModel): + def __init__(self, config): + super().__init__(config) + self.n_cameras = config.net.n_cameras + + # + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + state = rearrange(state, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + encoded_state = self.tokenizer.encode(state) + encoded_state = rearrange(encoded_state, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) * self.sigma_data + return encoded_state + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + latent = rearrange(latent, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + decoded_state = self.tokenizer.decode(latent / self.sigma_data) + decoded_state = rearrange(decoded_state, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + return decoded_state + + def add_condition_video_indicator_and_video_input_mask( + self, latent_state: torch.Tensor, condition: VideoExtendCondition, num_condition_t: Union[int, None] = None + ) -> VideoExtendCondition: + """Add condition_video_indicator and condition_video_input_mask to the condition object for video conditioning. + condition_video_indicator is a binary tensor indicating the condition region in the latent state. 1x1xTx1x1 tensor. + condition_video_input_mask will be concat with the input for the network. + Args: + latent_state (torch.Tensor): latent state tensor in shape B,C,T,H,W + condition (VideoExtendCondition): condition object + num_condition_t (int): number of condition latent T, used in inference to decide the condition region and config.conditioner.video_cond_bool.condition_location == "first_n" + Returns: + VideoExtendCondition: updated condition object + """ + T = latent_state.shape[2] + latent_dtype = latent_state.dtype + condition_video_indicator = torch.zeros(1, 1, T, 1, 1, device=latent_state.device).type( + latent_dtype + ) # 1 for condition region + + condition_video_indicator = rearrange( + condition_video_indicator, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras + ) + # Only in inference to decide the condition region + assert num_condition_t is not None, "num_condition_t should be provided" + assert num_condition_t <= T, f"num_condition_t should be less than T, get {num_condition_t}, {T}" + log.info( + f"condition_location first_n, num_condition_t {num_condition_t}, condition.video_cond_bool {condition.video_cond_bool}" + ) + condition_video_indicator[:, :, :num_condition_t] += 1.0 + condition_video_indicator = rearrange( + condition_video_indicator, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras + ) + + condition.gt_latent = latent_state + condition.condition_video_indicator = condition_video_indicator + + B, C, T, H, W = latent_state.shape + # Create additional input_mask channel, this will be concatenated to the input of the network + ones_padding = torch.ones((B, 1, T, H, W), dtype=latent_state.dtype, device=latent_state.device) + zeros_padding = torch.zeros((B, 1, T, H, W), dtype=latent_state.dtype, device=latent_state.device) + assert condition.video_cond_bool is not None, "video_cond_bool should be set" + + # The input mask indicate whether the input is conditional region or not + if condition.video_cond_bool: # Condition one given video frames + condition.condition_video_input_mask = ( + condition_video_indicator * ones_padding + (1 - condition_video_indicator) * zeros_padding + ) + else: # Unconditional case, use for cfg + condition.condition_video_input_mask = zeros_padding + + return condition diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_t2w.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_t2w.py new file mode 100644 index 00000000..659d15dc --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_t2w.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Callable, Dict, Optional, Tuple + +import torch +from cosmos1.models.diffusion.conditioner import CosmosCondition +from cosmos1.models.diffusion.diffusion.functional.batch_ops import batch_mul +from cosmos1.models.diffusion.diffusion.modules.denoiser_scaling import EDMScaling +from cosmos1.models.diffusion.diffusion.modules.res_sampler import COMMON_SOLVER_OPTIONS, Sampler +from cosmos1.models.diffusion.diffusion.types import DenoisePrediction +from cosmos1.models.diffusion.module.blocks import FourierFeatures +from cosmos1.models.diffusion.module.pretrained_vae import BaseVAE +from cosmos1.utils import log, misc +from cosmos1.utils.lazy_config import instantiate as lazy_instantiate +from torch import Tensor + + +class EDMSDE: + def __init__( + self, + sigma_max: float, + sigma_min: float, + ): + self.sigma_max = sigma_max + self.sigma_min = sigma_min + + +class DiffusionT2WModel(torch.nn.Module): + """Text-to-world diffusion model that generates video frames from text descriptions. + + This model implements a diffusion-based approach for generating videos conditioned on text input. + It handles the full pipeline including encoding/decoding through a VAE, diffusion sampling, + and classifier-free guidance. + """ + + def __init__(self, config): + """Initialize the diffusion model. + + Args: + config: Configuration object containing model parameters and architecture settings + """ + super().__init__() + # Initialize trained_data_record with defaultdict, key: image, video, iteration + self.config = config + + self.precision = { + "float32": torch.float32, + "float16": torch.float16, + "bfloat16": torch.bfloat16, + }[config.precision] + self.tensor_kwargs = {"device": "cuda", "dtype": self.precision} + log.debug(f"DiffusionModel: precision {self.precision}") + # Timer passed to network to detect slow ranks. + # 1. set data keys and data information + self.sigma_data = config.sigma_data + self.state_shape = list(config.latent_shape) + self.setup_data_key() + + # 2. setup up diffusion processing and scaling~(pre-condition), sampler + self.sde = EDMSDE(sigma_max=80, sigma_min=0.0002) + self.sampler = Sampler() + self.scaling = EDMScaling(self.sigma_data) + self.tokenizer = None + self.model = None + + @property + def net(self): + return self.model.net + + @property + def conditioner(self): + return self.model.conditioner + + @property + def logvar(self): + return self.model.logvar + + def set_up_tokenizer(self, tokenizer_dir: str): + self.tokenizer: BaseVAE = lazy_instantiate(self.config.tokenizer) + self.tokenizer.load_weights(tokenizer_dir) + if hasattr(self.tokenizer, "reset_dtype"): + self.tokenizer.reset_dtype() + + @misc.timer("DiffusionModel: set_up_model") + def set_up_model(self, memory_format: torch.memory_format = torch.preserve_format): + """Initialize the core model components including network, conditioner and logvar.""" + self.model = self.build_model() + self.model = self.model.to(memory_format=memory_format, **self.tensor_kwargs) + + def build_model(self) -> torch.nn.ModuleDict: + """Construct the model's neural network components. + + Returns: + ModuleDict containing the network, conditioner and logvar components + """ + config = self.config + net = lazy_instantiate(config.net) + conditioner = lazy_instantiate(config.conditioner) + logvar = torch.nn.Sequential( + FourierFeatures(num_channels=128, normalize=True), torch.nn.Linear(128, 1, bias=False) + ) + + return torch.nn.ModuleDict( + { + "net": net, + "conditioner": conditioner, + "logvar": logvar, + } + ) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + """Encode input state into latent representation using VAE. + + Args: + state: Input tensor to encode + + Returns: + Encoded latent representation scaled by sigma_data + """ + return self.tokenizer.encode(state) * self.sigma_data + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """Decode latent representation back to pixel space using VAE. + + Args: + latent: Latent tensor to decode + + Returns: + Decoded tensor in pixel space + """ + return self.tokenizer.decode(latent / self.sigma_data) + + def setup_data_key(self) -> None: + """Configure input data keys for video and image data.""" + self.input_data_key = self.config.input_data_key # by default it is video key for Video diffusion model + + def get_x0_fn_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + ) -> Callable: + """ + Generates a callable function `x0_fn` based on the provided data batch and guidance factor. + + This function processes the input data batch through a conditioning workflow to obtain + conditioned and unconditioned states. It then defines a nested function `x0_fn` which + applies denoising on an input `noise_x` at a given noise level `sigma`. + + Args: + data_batch: A batch of data used for conditioning. Format should align with conditioner + guidance: Scalar value that modulates influence of conditioned vs unconditioned state + is_negative_prompt: Use negative prompt t5 in uncondition if true + + Returns: + A function `x0_fn(noise_x, sigma)` that takes noise_x and sigma, returns x0 prediction + """ + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + cond_x0 = self.denoise(noise_x, sigma, condition).x0 + uncond_x0 = self.denoise(noise_x, sigma, uncondition).x0 + raw_x0 = cond_x0 + guidance * (cond_x0 - uncond_x0) + if "guided_image" in data_batch: + # replacement trick that enables inpainting with base model + assert "guided_mask" in data_batch, "guided_mask should be in data_batch if guided_image is present" + guide_image = data_batch["guided_image"] + guide_mask = data_batch["guided_mask"] + raw_x0 = guide_mask * guide_image + (1 - guide_mask) * raw_x0 + + return raw_x0 + + return x0_fn + + def denoise(self, xt: torch.Tensor, sigma: torch.Tensor, condition: CosmosCondition) -> DenoisePrediction: + """ + Performs denoising on the input noise data, noise level, and condition + + Args: + xt (torch.Tensor): The input noise data. + sigma (torch.Tensor): The noise level. + condition (CosmosCondition): conditional information, generated from self.conditioner + + Returns: + DenoisePrediction: The denoised prediction, it includes clean data predicton (x0), \ + noise prediction (eps_pred) and optional confidence (logvar). + """ + + xt = xt.to(**self.tensor_kwargs) + sigma = sigma.to(**self.tensor_kwargs) + # get precondition for the network + c_skip, c_out, c_in, c_noise = self.scaling(sigma=sigma) + + # forward pass through the network + net_output = self.net( + x=batch_mul(c_in, xt), # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + timesteps=c_noise, # Eq. 7 of https://arxiv.org/pdf/2206.00364.pdf + **condition.to_dict(), + ) + + logvar = self.model.logvar(c_noise) + x0_pred = batch_mul(c_skip, xt) + batch_mul(c_out, net_output) + + # get noise prediction based on sde + eps_pred = batch_mul(xt - x0_pred, 1.0 / sigma) + + return DenoisePrediction(x0_pred, eps_pred, logvar) + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + solver_option: COMMON_SOLVER_OPTIONS = "2ab", + x_sigma_max: Optional[torch.Tensor] = None, + sigma_max: float | None = None, + ) -> Tensor: + """Generate samples from a data batch using diffusion sampling. + + This function generates samples from either image or video data batches using diffusion sampling. + It handles both conditional and unconditional generation with classifier-free guidance. + + Args: + data_batch (Dict): Raw data batch from the training data loader + guidance (float, optional): Classifier-free guidance weight. Defaults to 1.5. + seed (int, optional): Random seed for reproducibility. Defaults to 1. + state_shape (Tuple | None, optional): Shape of the state tensor. Uses self.state_shape if None. Defaults to None. + n_sample (int | None, optional): Number of samples to generate. Defaults to None. + is_negative_prompt (bool, optional): Whether to use negative prompt for unconditional generation. Defaults to False. + num_steps (int, optional): Number of diffusion sampling steps. Defaults to 35. + solver_option (COMMON_SOLVER_OPTIONS, optional): Differential equation solver option. Defaults to "2ab" (multistep solver). + x_sigma_max (Optional[torch.Tensor], optional): Initial noisy tensor. If None, randomly initialized. Defaults to None. + sigma_max (float | None, optional): Maximum noise level. Uses self.sde.sigma_max if None. Defaults to None. + + Returns: + Tensor: Generated samples after diffusion sampling + """ + x0_fn = self.get_x0_fn_from_batch(data_batch, guidance, is_negative_prompt=is_negative_prompt) + if sigma_max is None: + sigma_max = self.sde.sigma_max + else: + log.info("Using provided sigma_max for diffusion sampling.") + if x_sigma_max is None: + x_sigma_max = ( + misc.arch_invariant_rand( + (n_sample,) + tuple(state_shape), + torch.float32, + self.tensor_kwargs["device"], + seed, + ) + * sigma_max + ) + + samples = self.sampler( + x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=sigma_max, solver_option=solver_option + ) + + return samples diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_v2w.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_v2w.py new file mode 100644 index 00000000..a4829719 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/model/model_v2w.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from dataclasses import dataclass +from typing import Callable, Dict, Optional, Tuple, Union + +import torch +from cosmos1.models.diffusion.conditioner import VideoExtendCondition +from cosmos1.models.diffusion.config.base.conditioner import VideoCondBoolConfig +from cosmos1.models.diffusion.diffusion.functional.batch_ops import batch_mul +from cosmos1.models.diffusion.model.model_t2w import DiffusionT2WModel +from cosmos1.utils import log, misc +from torch import Tensor + + +@dataclass +class VideoDenoisePrediction: + x0: torch.Tensor # clean data prediction + eps: Optional[torch.Tensor] = None # noise prediction + logvar: Optional[torch.Tensor] = None # log variance of noise prediction, can be used a confidence / uncertainty + xt: Optional[torch.Tensor] = None # input to the network, before muliply with c_in + x0_pred_replaced: Optional[torch.Tensor] = None # x0 prediction with condition region replaced by gt_latent + + +class DiffusionV2WModel(DiffusionT2WModel): + def __init__(self, config): + super().__init__(config) + + def augment_conditional_latent_frames( + self, + condition: VideoExtendCondition, + cfg_video_cond_bool: VideoCondBoolConfig, + gt_latent: Tensor, + condition_video_augment_sigma_in_inference: float = 0.001, + sigma: Tensor = None, + seed: int = 1, + ) -> Union[VideoExtendCondition, Tensor]: + """Augments the conditional frames with noise during inference. + + Args: + condition (VideoExtendCondition): condition object + condition_video_indicator: binary tensor indicating the region is condition(value=1) or generation(value=0). Bx1xTx1x1 tensor. + condition_video_input_mask: input mask for the network input, indicating the condition region. B,1,T,H,W tensor. will be concat with the input for the network. + cfg_video_cond_bool (VideoCondBoolConfig): video condition bool config + gt_latent (Tensor): ground truth latent tensor in shape B,C,T,H,W + condition_video_augment_sigma_in_inference (float): sigma for condition video augmentation in inference + sigma (Tensor): noise level for the generation region + seed (int): random seed for reproducibility + Returns: + VideoExtendCondition: updated condition object + condition_video_augment_sigma: sigma for the condition region, feed to the network + augment_latent (Tensor): augmented latent tensor in shape B,C,T,H,W + + """ + + # Inference only, use fixed sigma for the condition region + assert condition_video_augment_sigma_in_inference is not None, ( + "condition_video_augment_sigma_in_inference should be provided" + ) + augment_sigma = condition_video_augment_sigma_in_inference + + if augment_sigma >= sigma.flatten()[0]: + # This is a inference trick! If the sampling sigma is smaller than the augment sigma, we will start denoising the condition region together. + # This is achieved by setting all region as `generation`, i.e. value=0 + log.debug("augment_sigma larger than sigma or other frame, remove condition") + condition.condition_video_indicator = condition.condition_video_indicator * 0 + + augment_sigma = torch.tensor([augment_sigma], **self.tensor_kwargs) + + # Now apply the augment_sigma to the gt_latent + + noise = misc.arch_invariant_rand( + gt_latent.shape, + torch.float32, + self.tensor_kwargs["device"], + seed, + ) + + augment_latent = gt_latent + noise * augment_sigma[:, None, None, None, None] + + _, _, c_in_augment, _ = self.scaling(sigma=augment_sigma) + + # Multiply the whole latent with c_in_augment + augment_latent_cin = batch_mul(augment_latent, c_in_augment) + + # Since the whole latent will multiply with c_in later, we devide the value to cancel the effect + _, _, c_in, _ = self.scaling(sigma=sigma) + augment_latent_cin = batch_mul(augment_latent_cin, 1 / c_in) + + return condition, augment_latent_cin + + def denoise( + self, + noise_x: Tensor, + sigma: Tensor, + condition: VideoExtendCondition, + condition_video_augment_sigma_in_inference: float = 0.001, + seed: int = 1, + ) -> VideoDenoisePrediction: + """Denoises input tensor using conditional video generation. + + Args: + noise_x (Tensor): Noisy input tensor. + sigma (Tensor): Noise level. + condition (VideoExtendCondition): Condition for denoising. + condition_video_augment_sigma_in_inference (float): sigma for condition video augmentation in inference + seed (int): Random seed for reproducibility + Returns: + VideoDenoisePrediction containing: + - x0: Denoised prediction + - eps: Noise prediction + - logvar: Log variance of noise prediction + - xt: Input before c_in multiplication + - x0_pred_replaced: x0 prediction with condition regions replaced by ground truth + """ + + assert condition.gt_latent is not None, ( + f"find None gt_latent in condition, likely didn't call self.add_condition_video_indicator_and_video_input_mask when preparing the condition or this is a image batch but condition.data_type is wrong, get {noise_x.shape}" + ) + gt_latent = condition.gt_latent + cfg_video_cond_bool: VideoCondBoolConfig = self.config.conditioner.video_cond_bool + + condition_latent = gt_latent + + # Augment the latent with different sigma value, and add the augment_sigma to the condition object if needed + condition, augment_latent = self.augment_conditional_latent_frames( + condition, cfg_video_cond_bool, condition_latent, condition_video_augment_sigma_in_inference, sigma, seed + ) + condition_video_indicator = condition.condition_video_indicator # [B, 1, T, 1, 1] + # Compose the model input with condition region (augment_latent) and generation region (noise_x) + new_noise_xt = condition_video_indicator * augment_latent + (1 - condition_video_indicator) * noise_x + # Call the base model + denoise_pred = super().denoise(new_noise_xt, sigma, condition) + + x0_pred_replaced = condition_video_indicator * gt_latent + (1 - condition_video_indicator) * denoise_pred.x0 + + x0_pred = x0_pred_replaced + + return VideoDenoisePrediction( + x0=x0_pred, + eps=batch_mul(noise_x - x0_pred, 1.0 / sigma), + logvar=denoise_pred.logvar, + xt=new_noise_xt, + x0_pred_replaced=x0_pred_replaced, + ) + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + condition_latent: Union[torch.Tensor, None] = None, + num_condition_t: Union[int, None] = None, + condition_video_augment_sigma_in_inference: float = None, + add_input_frames_guidance: bool = False, + x_sigma_max: Optional[torch.Tensor] = None, + ) -> Tensor: + """Generates video samples conditioned on input frames. + + Args: + data_batch: Input data dictionary + guidance: Classifier-free guidance scale + seed: Random seed for reproducibility + state_shape: Shape of output tensor (defaults to model's state shape) + n_sample: Number of samples to generate (defaults to batch size) + is_negative_prompt: Whether to use negative prompting + num_steps: Number of denoising steps + condition_latent: Conditioning frames tensor (B,C,T,H,W) + num_condition_t: Number of frames to condition on + condition_video_augment_sigma_in_inference: Noise level for condition augmentation + add_input_frames_guidance: Whether to apply guidance to input frames + x_sigma_max: Maximum noise level tensor + + Returns: + Generated video samples tensor + """ + + if n_sample is None: + input_key = self.input_data_key + n_sample = data_batch[input_key].shape[0] + if state_shape is None: + log.debug(f"Default Video state shape is used. {self.state_shape}") + state_shape = self.state_shape + + assert condition_latent is not None, "condition_latent should be provided" + + x0_fn = self.get_x0_fn_from_batch_with_condition_latent( + data_batch, + guidance, + is_negative_prompt=is_negative_prompt, + condition_latent=condition_latent, + num_condition_t=num_condition_t, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + add_input_frames_guidance=add_input_frames_guidance, + seed=seed, + ) + if x_sigma_max is None: + x_sigma_max = ( + misc.arch_invariant_rand( + (n_sample,) + tuple(state_shape), + torch.float32, + self.tensor_kwargs["device"], + seed, + ) + * self.sde.sigma_max + ) + + samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=self.sde.sigma_max) + return samples + + def get_x0_fn_from_batch_with_condition_latent( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + condition_latent: torch.Tensor = None, + num_condition_t: Union[int, None] = None, + condition_video_augment_sigma_in_inference: float = None, + add_input_frames_guidance: bool = False, + seed: int = 1, + ) -> Callable: + """Creates denoising function for conditional video generation. + + Args: + data_batch: Input data dictionary + guidance: Classifier-free guidance scale + is_negative_prompt: Whether to use negative prompting + condition_latent: Conditioning frames tensor (B,C,T,H,W) + num_condition_t: Number of frames to condition on + condition_video_augment_sigma_in_inference: Noise level for condition augmentation + add_input_frames_guidance: Whether to apply guidance to input frames + seed: Random seed for reproducibility + + Returns: + Function that takes noisy input and noise level and returns denoised prediction + """ + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + + condition.video_cond_bool = True + condition = self.add_condition_video_indicator_and_video_input_mask( + condition_latent, condition, num_condition_t + ) + + uncondition.video_cond_bool = False if add_input_frames_guidance else True + uncondition = self.add_condition_video_indicator_and_video_input_mask( + condition_latent, uncondition, num_condition_t + ) + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + cond_x0 = self.denoise( + noise_x, + sigma, + condition, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + seed=seed, + ).x0_pred_replaced + uncond_x0 = self.denoise( + noise_x, + sigma, + uncondition, + condition_video_augment_sigma_in_inference=condition_video_augment_sigma_in_inference, + seed=seed, + ).x0_pred_replaced + + return cond_x0 + guidance * (cond_x0 - uncond_x0) + + return x0_fn + + def add_condition_video_indicator_and_video_input_mask( + self, latent_state: torch.Tensor, condition: VideoExtendCondition, num_condition_t: Union[int, None] = None + ) -> VideoExtendCondition: + """Adds conditioning masks to VideoExtendCondition object. + + Creates binary indicators and input masks for conditional video generation. + + Args: + latent_state: Input latent tensor (B,C,T,H,W) + condition: VideoExtendCondition object to update + num_condition_t: Number of frames to condition on + + Returns: + Updated VideoExtendCondition with added masks: + - condition_video_indicator: Binary tensor marking condition regions + - condition_video_input_mask: Input mask for network + - gt_latent: Ground truth latent tensor + """ + T = latent_state.shape[2] + latent_dtype = latent_state.dtype + condition_video_indicator = torch.zeros(1, 1, T, 1, 1, device=latent_state.device).type( + latent_dtype + ) # 1 for condition region + + # Only in inference to decide the condition region + assert num_condition_t is not None, "num_condition_t should be provided" + assert num_condition_t <= T, f"num_condition_t should be less than T, get {num_condition_t}, {T}" + log.debug( + f"condition_location first_n, num_condition_t {num_condition_t}, condition.video_cond_bool {condition.video_cond_bool}" + ) + condition_video_indicator[:, :, :num_condition_t] += 1.0 + + condition.gt_latent = latent_state + condition.condition_video_indicator = condition_video_indicator + + B, C, T, H, W = latent_state.shape + # Create additional input_mask channel, this will be concatenated to the input of the network + ones_padding = torch.ones((B, 1, T, H, W), dtype=latent_state.dtype, device=latent_state.device) + zeros_padding = torch.zeros((B, 1, T, H, W), dtype=latent_state.dtype, device=latent_state.device) + assert condition.video_cond_bool is not None, "video_cond_bool should be set" + + # The input mask indicate whether the input is conditional region or not + if condition.video_cond_bool: # Condition one given video frames + condition.condition_video_input_mask = ( + condition_video_indicator * ones_padding + (1 - condition_video_indicator) * zeros_padding + ) + else: # Unconditional case, use for cfg + condition.condition_video_input_mask = zeros_padding + + return condition diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/attention.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/attention.py new file mode 100644 index 00000000..215bc9d1 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/attention.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import List, Optional + +import numpy as np +import torch +import transformer_engine as te +from einops import rearrange +from torch import nn +from torch.utils.checkpoint import checkpoint +from transformer_engine.pytorch.attention import DotProductAttention, apply_rotary_pos_emb + + +# ---------------------- Feed Forward Network ----------------------- + + +class FeedForward(nn.Module): + """ + Transformer FFN with optional gating + + Parameters: + d_model (int): Dimensionality of input features. + d_ff (int): Dimensionality of the hidden layer. + dropout (float, optional): Dropout rate applied after the activation function. Defaults to 0.1. + activation (callable, optional): The activation function applied after the first linear layer. + Defaults to nn.ReLU(). + is_gated (bool, optional): If set to True, incorporates gating mechanism to the feed-forward layer. + Defaults to False. + bias (bool, optional): If set to True, adds a bias to the linear layers. Defaults to True. + + Example: + >>> ff = FeedForward(d_model=512, d_ff=2048) + >>> x = torch.randn(64, 10, 512) # Example input tensor + >>> output = ff(x) + >>> print(output.shape) # Expected shape: (64, 10, 512) + """ + + def __init__( + self, + d_model: int, + d_ff: int, + dropout: float = 0.1, + activation=nn.ReLU(), + is_gated: bool = False, + bias: bool = False, + ) -> None: + super().__init__() + + self.layer1 = nn.Linear(d_model, d_ff, bias=bias) + self.layer2 = nn.Linear(d_ff, d_model, bias=bias) + + self.dropout = nn.Dropout(dropout) + self.activation = activation + self.is_gated = is_gated + if is_gated: + self.linear_gate = nn.Linear(d_model, d_ff, bias=False) + + def forward(self, x: torch.Tensor): + g = self.activation(self.layer1(x)) + if self.is_gated: + x = g * self.linear_gate(x) + else: + x = g + assert self.dropout.p == 0.0, "we skip dropout" + return self.layer2(x) + + +class GPT2FeedForward(FeedForward): + def __init__(self, d_model: int, d_ff: int, dropout: float = 0.1, bias: bool = False): + super().__init__( + d_model=d_model, + d_ff=d_ff, + dropout=dropout, + activation=nn.GELU(), + is_gated=False, + bias=bias, + ) + + def forward(self, x: torch.Tensor): + assert self.dropout.p == 0.0, "we skip dropout" + + x = self.layer1(x) + + def activation_layer2_forward(x): + x = self.activation(x) + x = self.layer2(x) + return x + + x = checkpoint(activation_layer2_forward, x, use_reentrant=False) + return x + + +# ---------------------- Normalization Layer ----------------------- + + +def normalize(x: torch.Tensor, dim: Optional[List[int]] = None, eps: float = 0) -> torch.Tensor: + """ + Normalizes the input tensor along specified dimensions such that the average square norm of elements is adjusted. + + Args: + x (torch.Tensor): The input tensor to normalize. + dim (list, optional): The dimensions over which to normalize. If None, normalizes over all dimensions except the first. + eps (float, optional): A small constant to ensure numerical stability during division. + + Returns: + torch.Tensor: The normalized tensor. + """ + if dim is None: + dim = list(range(1, x.ndim)) + norm = torch.linalg.vector_norm(x, dim=dim, keepdim=True, dtype=torch.float32) + norm = torch.add(eps, norm, alpha=np.sqrt(norm.numel() / x.numel())) + return x / norm.to(x.dtype) + + +def get_normalization(name: str, channels: int): + if name == "I": + return nn.Identity() + elif name == "R": + return te.pytorch.RMSNorm(channels, eps=1e-6) + else: + raise ValueError(f"Normalization {name} not found") + + +class BaseAttentionOp(nn.Module): + def __init__(self): + super().__init__() + + +class Attention(nn.Module): + """ + Generalized attention impl. + + Allowing for both self-attention and cross-attention configurations depending on whether a `context_dim` is provided. + If `context_dim` is None, self-attention is assumed. + + Parameters: + query_dim (int): Dimension of each query vector. + context_dim (int, optional): Dimension of each context vector. If None, self-attention is assumed. + heads (int, optional): Number of attention heads. Defaults to 8. + dim_head (int, optional): Dimension of each head. Defaults to 64. + dropout (float, optional): Dropout rate applied to the output of the attention block. Defaults to 0.0. + attn_op (BaseAttentionOp, optional): Custom attention operation to be used instead of the default. + qkv_bias (bool, optional): If True, adds a learnable bias to query, key, and value projections. Defaults to False. + out_bias (bool, optional): If True, adds a learnable bias to the output projection. Defaults to False. + qkv_norm (str, optional): A string representing normalization strategies for query, key, and value projections. + Defaults to "SSI". + qkv_norm_mode (str, optional): A string representing normalization mode for query, key, and value projections. + Defaults to 'per_head'. Only support 'per_head'. + + Examples: + >>> attn = Attention(query_dim=128, context_dim=256, heads=4, dim_head=32, dropout=0.1) + >>> query = torch.randn(10, 128) # Batch size of 10 + >>> context = torch.randn(10, 256) # Batch size of 10 + >>> output = attn(query, context) # Perform the attention operation + + Note: + https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223 + """ + + def __init__( + self, + query_dim: int, + context_dim=None, + heads=8, + dim_head=64, + dropout=0.0, + attn_op: Optional[BaseAttentionOp] = None, + qkv_bias: bool = False, + out_bias: bool = False, + qkv_norm: str = "SSI", + qkv_norm_mode: str = "per_head", + backend: str = "transformer_engine", + qkv_format: str = "bshd", + ) -> None: + super().__init__() + + self.is_selfattn = context_dim is None # self attention + + inner_dim = dim_head * heads + context_dim = query_dim if context_dim is None else context_dim + + self.heads = heads + self.dim_head = dim_head + self.qkv_norm_mode = qkv_norm_mode + self.qkv_format = qkv_format + + if self.qkv_norm_mode == "per_head": + norm_dim = dim_head + else: + raise ValueError(f"Normalization mode {self.qkv_norm_mode} not found, only support 'per_head'") + + self.backend = backend + + self.to_q = nn.Sequential( + nn.Linear(query_dim, inner_dim, bias=qkv_bias), + get_normalization(qkv_norm[0], norm_dim), + ) + self.to_k = nn.Sequential( + nn.Linear(context_dim, inner_dim, bias=qkv_bias), + get_normalization(qkv_norm[1], norm_dim), + ) + self.to_v = nn.Sequential( + nn.Linear(context_dim, inner_dim, bias=qkv_bias), + get_normalization(qkv_norm[2], norm_dim), + ) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim, bias=out_bias), + nn.Dropout(dropout), + ) + + if attn_op: # use what is given + self.attn_op = attn_op + elif self.backend == "transformer_engine": + sequence_parallel = False + self.attn_op: BaseAttentionOp = DotProductAttention( + self.heads, + self.dim_head, + num_gqa_groups=self.heads, + attention_dropout=0, + qkv_format=qkv_format, + attn_mask_type="no_mask", + tp_size=1, + tp_group=None, + sequence_parallel=sequence_parallel, + ) + else: + raise ValueError(f"Backend {backend} not found") + + def cal_qkv( + self, x, context=None, mask=None, rope_emb=None, **kwargs + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del kwargs + + """ + self.to_q, self.to_k, self.to_v are nn.Sequential with projection + normalization layers. + Before 07/24/2024, these modules normalize across all heads. + After 07/24/2024, to support tensor parallelism and follow the common practice in the community, + we support to normalize per head. + To keep the checkpoint copatibility with the previous code, + we keep the nn.Sequential but call the projection and the normalization layers separately. + We use a flag `self.qkv_norm_mode` to control the normalization behavior. + The default value of `self.qkv_norm_mode` is "per_head", which means we normalize per head. + """ + if self.qkv_norm_mode == "per_head": + q = self.to_q[0](x) + context = x if context is None else context + k = self.to_k[0](context) + v = self.to_v[0](context) + q, k, v = map( + lambda t: rearrange(t, "b ... (n c) -> b ... n c", n=self.heads, c=self.dim_head), + (q, k, v), + ) + else: + raise ValueError(f"Normalization mode {self.qkv_norm_mode} not found, only support 'per_head'") + + q = self.to_q[1](q) + k = self.to_k[1](k) + v = self.to_v[1](v) + if self.is_selfattn and rope_emb is not None: # only apply to self-attention! + q = apply_rotary_pos_emb(q, rope_emb, tensor_format=self.qkv_format, fused=True) + k = apply_rotary_pos_emb(k, rope_emb, tensor_format=self.qkv_format, fused=True) + return q, k, v + + def cal_attn(self, q, k, v, mask=None): + if self.backend == "transformer_engine": + seq_dim = self.qkv_format.index("s") + assert q.shape[seq_dim] > 1 and k.shape[seq_dim] > 1, ( + "Seqlen must be larger than 1 for TE Attention starting with 1.8 TE version." + ) + out = self.attn_op(q, k, v, core_attention_bias_type="no_bias", core_attention_bias=None) # [B, Mq, H, V] + return self.to_out(out) + elif self.backend == "torch": + out = self.attn_op(q, k, v, mask=mask) # [B, Mq, H, V] + return self.to_out(rearrange(out, " b ... n c -> b ... (n c)")) + else: + raise ValueError(f"Backend {self.backend} not found") + + def forward( + self, + x, + context=None, + mask=None, + rope_emb=None, + **kwargs, + ): + """ + Args: + x (Tensor): The query tensor of shape [B, Mq, K] + context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None + """ + q, k, v = self.cal_qkv(x, context, mask, rope_emb=rope_emb, **kwargs) + return self.cal_attn(q, k, v, mask) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/blocks.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/blocks.py new file mode 100644 index 00000000..ef41903d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/blocks.py @@ -0,0 +1,559 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math +from typing import Optional + +import numpy as np +import torch +from cosmos1.models.diffusion.module.attention import Attention, GPT2FeedForward +from cosmos1.utils import log +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from torch import nn + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class Timesteps(nn.Module): + def __init__(self, num_channels): + super().__init__() + self.num_channels = num_channels + + def forward(self, timesteps): + in_dype = timesteps.dtype + half_dim = self.num_channels // 2 + exponent = -math.log(10000) * torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) + exponent = exponent / (half_dim - 0.0) + + emb = torch.exp(exponent) + emb = timesteps[:, None].float() * emb[None, :] + + sin_emb = torch.sin(emb) + cos_emb = torch.cos(emb) + emb = torch.cat([cos_emb, sin_emb], dim=-1) + + return emb.to(in_dype) + + +class TimestepEmbedding(nn.Module): + def __init__(self, in_features: int, out_features: int, use_adaln_lora: bool = False): + super().__init__() + log.debug( + f"Using AdaLN LoRA Flag: {use_adaln_lora}. We enable bias if no AdaLN LoRA for backward compatibility." + ) + self.linear_1 = nn.Linear(in_features, out_features, bias=not use_adaln_lora) + self.activation = nn.SiLU() + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.linear_2 = nn.Linear(out_features, 3 * out_features, bias=False) + else: + self.linear_2 = nn.Linear(out_features, out_features, bias=True) + + def forward(self, sample: torch.Tensor) -> torch.Tensor: + emb = self.linear_1(sample) + emb = self.activation(emb) + emb = self.linear_2(emb) + + if self.use_adaln_lora: + adaln_lora_B_3D = emb + emb_B_D = sample + else: + emb_B_D = emb + adaln_lora_B_3D = None + + return emb_B_D, adaln_lora_B_3D + + +class FourierFeatures(nn.Module): + """ + Implements a layer that generates Fourier features from input tensors, based on randomly sampled + frequencies and phases. This can help in learning high-frequency functions in low-dimensional problems. + + [B] -> [B, D] + + Parameters: + num_channels (int): The number of Fourier features to generate. + bandwidth (float, optional): The scaling factor for the frequency of the Fourier features. Defaults to 1. + normalize (bool, optional): If set to True, the outputs are scaled by sqrt(2), usually to normalize + the variance of the features. Defaults to False. + + Example: + >>> layer = FourierFeatures(num_channels=256, bandwidth=0.5, normalize=True) + >>> x = torch.randn(10, 256) # Example input tensor + >>> output = layer(x) + >>> print(output.shape) # Expected shape: (10, 256) + """ + + def __init__(self, num_channels, bandwidth=1, normalize=False): + super().__init__() + self.register_buffer("freqs", 2 * np.pi * bandwidth * torch.randn(num_channels), persistent=True) + self.register_buffer("phases", 2 * np.pi * torch.rand(num_channels), persistent=True) + self.gain = np.sqrt(2) if normalize else 1 + + def forward(self, x, gain: float = 1.0): + """ + Apply the Fourier feature transformation to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + gain (float, optional): An additional gain factor applied during the forward pass. Defaults to 1. + + Returns: + torch.Tensor: The transformed tensor, with Fourier features applied. + """ + in_dtype = x.dtype + x = x.to(torch.float32).ger(self.freqs.to(torch.float32)).add(self.phases.to(torch.float32)) + x = x.cos().mul(self.gain * gain).to(in_dtype) + return x + + +class PatchEmbed(nn.Module): + """ + PatchEmbed is a module for embedding patches from an input tensor by applying either 3D or 2D convolutional layers, + depending on the . This module can process inputs with temporal (video) and spatial (image) dimensions, + making it suitable for video and image processing tasks. It supports dividing the input into patches + and embedding each patch into a vector of size `out_channels`. + + Parameters: + - spatial_patch_size (int): The size of each spatial patch. + - temporal_patch_size (int): The size of each temporal patch. + - in_channels (int): Number of input channels. Default: 3. + - out_channels (int): The dimension of the embedding vector for each patch. Default: 768. + - bias (bool): If True, adds a learnable bias to the output of the convolutional layers. Default: True. + """ + + def __init__( + self, + spatial_patch_size, + temporal_patch_size, + in_channels=3, + out_channels=768, + bias=True, + ): + super().__init__() + self.spatial_patch_size = spatial_patch_size + self.temporal_patch_size = temporal_patch_size + + self.proj = nn.Sequential( + Rearrange( + "b c (t r) (h m) (w n) -> b t h w (c r m n)", + r=temporal_patch_size, + m=spatial_patch_size, + n=spatial_patch_size, + ), + nn.Linear( + in_channels * spatial_patch_size * spatial_patch_size * temporal_patch_size, out_channels, bias=bias + ), + ) + self.out = nn.Identity() + + def forward(self, x): + """ + Forward pass of the PatchEmbed module. + + Parameters: + - x (torch.Tensor): The input tensor of shape (B, C, T, H, W) where + B is the batch size, + C is the number of channels, + T is the temporal dimension, + H is the height, and + W is the width of the input. + + Returns: + - torch.Tensor: The embedded patches as a tensor, with shape b t h w c. + """ + assert x.dim() == 5 + _, _, T, H, W = x.shape + assert H % self.spatial_patch_size == 0 and W % self.spatial_patch_size == 0 + assert T % self.temporal_patch_size == 0 + x = self.proj(x) + return self.out(x) + + +class FinalLayer(nn.Module): + """ + The final layer of video DiT. + """ + + def __init__( + self, + hidden_size, + spatial_patch_size, + temporal_patch_size, + out_channels, + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + ): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear( + hidden_size, spatial_patch_size * spatial_patch_size * temporal_patch_size * out_channels, bias=False + ) + self.hidden_size = hidden_size + self.n_adaln_chunks = 2 + self.use_adaln_lora = use_adaln_lora + if use_adaln_lora: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, adaln_lora_dim, bias=False), + nn.Linear(adaln_lora_dim, self.n_adaln_chunks * hidden_size, bias=False), + ) + else: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, self.n_adaln_chunks * hidden_size, bias=False) + ) + + def forward( + self, + x_BT_HW_D, + emb_B_D, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ): + if self.use_adaln_lora: + assert adaln_lora_B_3D is not None + shift_B_D, scale_B_D = (self.adaLN_modulation(emb_B_D) + adaln_lora_B_3D[:, : 2 * self.hidden_size]).chunk( + 2, dim=1 + ) + else: + shift_B_D, scale_B_D = self.adaLN_modulation(emb_B_D).chunk(2, dim=1) + + B = emb_B_D.shape[0] + T = x_BT_HW_D.shape[0] // B + shift_BT_D, scale_BT_D = repeat(shift_B_D, "b d -> (b t) d", t=T), repeat(scale_B_D, "b d -> (b t) d", t=T) + x_BT_HW_D = modulate(self.norm_final(x_BT_HW_D), shift_BT_D, scale_BT_D) + + x_BT_HW_D = self.linear(x_BT_HW_D) + return x_BT_HW_D + + +class VideoAttn(nn.Module): + """ + Implements video attention with optional cross-attention capabilities. + + This module processes video features while maintaining their spatio-temporal structure. It can perform + self-attention within the video features or cross-attention with external context features. + + Parameters: + x_dim (int): Dimension of input feature vectors + context_dim (Optional[int]): Dimension of context features for cross-attention. None for self-attention + num_heads (int): Number of attention heads + bias (bool): Whether to include bias in attention projections. Default: False + qkv_norm_mode (str): Normalization mode for query/key/value projections. Must be "per_head". Default: "per_head" + x_format (str): Format of input tensor. Must be "BTHWD". Default: "BTHWD" + n_cameras (int): Extra parameter used in multi-view diffusion model. It indicated total number of camera we model together. + + Input shape: + - x: (T, H, W, B, D) video features + - context (optional): (M, B, D) context features for cross-attention + where: + T: temporal dimension + H: height + W: width + B: batch size + D: feature dimension + M: context sequence length + """ + + def __init__( + self, + x_dim: int, + context_dim: Optional[int], + num_heads: int, + bias: bool = False, + qkv_norm_mode: str = "per_head", + x_format: str = "BTHWD", + n_cameras: int = 1, + ) -> None: + super().__init__() + self.x_format = x_format + self.n_cameras = n_cameras + self.attn = Attention( + x_dim, + context_dim, + num_heads, + x_dim // num_heads, + qkv_bias=bias, + qkv_norm="RRI", + out_bias=bias, + qkv_norm_mode=qkv_norm_mode, + qkv_format="sbhd", + ) + + def forward( + self, + x: torch.Tensor, + context: Optional[torch.Tensor] = None, + crossattn_mask: Optional[torch.Tensor] = None, + rope_emb_L_1_1_D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Forward pass for video attention. + + Args: + x (Tensor): Input tensor of shape (B, T, H, W, D) or (T, H, W, B, D) representing batches of video data. + context (Tensor): Context tensor of shape (B, M, D) or (M, B, D), + where M is the sequence length of the context. + crossattn_mask (Optional[Tensor]): An optional mask for cross-attention mechanisms. + rope_emb_L_1_1_D (Optional[Tensor]): + Rotary positional embedding tensor of shape (L, 1, 1, D). L == THW for current video training. + + Returns: + Tensor: The output tensor with applied attention, maintaining the input shape. + """ + if context is not None and self.n_cameras > 1: + x_T_H_W_B_D = rearrange(x, "(v t) h w b d -> t h w (v b) d", v=self.n_cameras) + context_M_B_D = rearrange(context, "(v m) b d -> m (v b) d", v=self.n_cameras) + else: + x_T_H_W_B_D = x + context_M_B_D = context + T, H, W, B, D = x_T_H_W_B_D.shape + x_THW_B_D = rearrange(x_T_H_W_B_D, "t h w b d -> (t h w) b d") + x_THW_B_D = self.attn( + x_THW_B_D, + context_M_B_D, + crossattn_mask, + rope_emb=rope_emb_L_1_1_D, + ) + x_T_H_W_B_D = rearrange(x_THW_B_D, "(t h w) b d -> t h w b d", h=H, w=W) + if context is not None and self.n_cameras > 1: + x_T_H_W_B_D = rearrange(x_T_H_W_B_D, "t h w (v b) d -> (v t) h w b d", v=self.n_cameras) + return x_T_H_W_B_D + + +def adaln_norm_state(norm_state, x, scale, shift): + normalized = norm_state(x) + return normalized * (1 + scale) + shift + + +class DITBuildingBlock(nn.Module): + """ + A building block for the DiT (Diffusion Transformer) architecture that supports different types of + attention and MLP operations with adaptive layer normalization. + + Parameters: + block_type (str): Type of block - one of: + - "cross_attn"/"ca": Cross-attention + - "full_attn"/"fa": Full self-attention + - "mlp"/"ff": MLP/feedforward block + x_dim (int): Dimension of input features + context_dim (Optional[int]): Dimension of context features for cross-attention + num_heads (int): Number of attention heads + mlp_ratio (float): MLP hidden dimension multiplier. Default: 4.0 + bias (bool): Whether to use bias in layers. Default: False + mlp_dropout (float): Dropout rate for MLP. Default: 0.0 + qkv_norm_mode (str): QKV normalization mode. Default: "per_head" + x_format (str): Input tensor format. Default: "BTHWD" + use_adaln_lora (bool): Whether to use AdaLN-LoRA. Default: False + adaln_lora_dim (int): Dimension for AdaLN-LoRA. Default: 256 + n_cameras (int): Extra parameter used in multi-view diffusion model. It indicated total number of camera we model together. + """ + + def __init__( + self, + block_type: str, + x_dim: int, + context_dim: Optional[int], + num_heads: int, + mlp_ratio: float = 4.0, + bias: bool = False, + mlp_dropout: float = 0.0, + qkv_norm_mode: str = "per_head", + x_format: str = "BTHWD", + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + n_cameras: int = 1, + ) -> None: + block_type = block_type.lower() + + super().__init__() + self.x_format = x_format + if block_type in ["cross_attn", "ca"]: + self.block = VideoAttn( + x_dim, + context_dim, + num_heads, + bias=bias, + qkv_norm_mode=qkv_norm_mode, + x_format=self.x_format, + n_cameras=n_cameras, + ) + elif block_type in ["full_attn", "fa"]: + self.block = VideoAttn( + x_dim, None, num_heads, bias=bias, qkv_norm_mode=qkv_norm_mode, x_format=self.x_format + ) + elif block_type in ["mlp", "ff"]: + self.block = GPT2FeedForward(x_dim, int(x_dim * mlp_ratio), dropout=mlp_dropout, bias=bias) + else: + raise ValueError(f"Unknown block type: {block_type}") + + self.block_type = block_type + self.use_adaln_lora = use_adaln_lora + + self.norm_state = nn.LayerNorm(x_dim, elementwise_affine=False, eps=1e-6) + self.n_adaln_chunks = 3 + if use_adaln_lora: + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(x_dim, adaln_lora_dim, bias=False), + nn.Linear(adaln_lora_dim, self.n_adaln_chunks * x_dim, bias=False), + ) + else: + self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(x_dim, self.n_adaln_chunks * x_dim, bias=False)) + + def forward( + self, + x: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + rope_emb_L_1_1_D: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Forward pass for dynamically configured blocks with adaptive normalization. + + Args: + x (Tensor): Input tensor of shape (B, T, H, W, D) or (T, H, W, B, D). + emb_B_D (Tensor): Embedding tensor for adaptive layer normalization modulation. + crossattn_emb (Tensor): Tensor for cross-attention blocks. + crossattn_mask (Optional[Tensor]): Optional mask for cross-attention. + rope_emb_L_1_1_D (Optional[Tensor]): + Rotary positional embedding tensor of shape (L, 1, 1, D). L == THW for current video training. + + Returns: + Tensor: The output tensor after processing through the configured block and adaptive normalization. + """ + if self.use_adaln_lora: + shift_B_D, scale_B_D, gate_B_D = (self.adaLN_modulation(emb_B_D) + adaln_lora_B_3D).chunk( + self.n_adaln_chunks, dim=1 + ) + else: + shift_B_D, scale_B_D, gate_B_D = self.adaLN_modulation(emb_B_D).chunk(self.n_adaln_chunks, dim=1) + + shift_1_1_1_B_D, scale_1_1_1_B_D, gate_1_1_1_B_D = ( + shift_B_D.unsqueeze(0).unsqueeze(0).unsqueeze(0), + scale_B_D.unsqueeze(0).unsqueeze(0).unsqueeze(0), + gate_B_D.unsqueeze(0).unsqueeze(0).unsqueeze(0), + ) + + if self.block_type in ["mlp", "ff"]: + x = x + gate_1_1_1_B_D * self.block( + adaln_norm_state(self.norm_state, x, scale_1_1_1_B_D, shift_1_1_1_B_D), + ) + elif self.block_type in ["full_attn", "fa"]: + x = x + gate_1_1_1_B_D * self.block( + adaln_norm_state(self.norm_state, x, scale_1_1_1_B_D, shift_1_1_1_B_D), + context=None, + rope_emb_L_1_1_D=rope_emb_L_1_1_D, + ) + elif self.block_type in ["cross_attn", "ca"]: + x = x + gate_1_1_1_B_D * self.block( + adaln_norm_state(self.norm_state, x, scale_1_1_1_B_D, shift_1_1_1_B_D), + context=crossattn_emb, + crossattn_mask=crossattn_mask, + rope_emb_L_1_1_D=rope_emb_L_1_1_D, + ) + else: + raise ValueError(f"Unknown block type: {self.block_type}") + + return x + + +class GeneralDITTransformerBlock(nn.Module): + """ + A wrapper module that manages a sequence of DITBuildingBlocks to form a complete transformer layer. + Each block in the sequence is specified by a block configuration string. + + Parameters: + x_dim (int): Dimension of input features + context_dim (int): Dimension of context features for cross-attention blocks + num_heads (int): Number of attention heads + block_config (str): String specifying block sequence (e.g. "ca-fa-mlp" for cross-attention, + full-attention, then MLP) + mlp_ratio (float): MLP hidden dimension multiplier. Default: 4.0 + x_format (str): Input tensor format. Default: "BTHWD" + use_adaln_lora (bool): Whether to use AdaLN-LoRA. Default: False + adaln_lora_dim (int): Dimension for AdaLN-LoRA. Default: 256 + n_cameras (int): Extra parameter used in multi-view diffusion model. Default: 1 + + The block_config string uses "-" to separate block types: + - "ca"/"cross_attn": Cross-attention block + - "fa"/"full_attn": Full self-attention block + - "mlp"/"ff": MLP/feedforward block + + Example: + block_config = "ca-fa-mlp" creates a sequence of: + 1. Cross-attention block + 2. Full self-attention block + 3. MLP block + """ + + def __init__( + self, + x_dim: int, + context_dim: int, + num_heads: int, + block_config: str, + mlp_ratio: float = 4.0, + x_format: str = "BTHWD", + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + n_cameras: int = 1, + ): + super().__init__() + self.blocks = nn.ModuleList() + self.x_format = x_format + for block_type in block_config.split("-"): + self.blocks.append( + DITBuildingBlock( + block_type, + x_dim, + context_dim, + num_heads, + mlp_ratio, + x_format=self.x_format, + use_adaln_lora=use_adaln_lora, + adaln_lora_dim=adaln_lora_dim, + n_cameras=n_cameras, + ) + ) + + def forward( + self, + x: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + rope_emb_L_1_1_D: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + extra_per_block_pos_emb: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if extra_per_block_pos_emb is not None: + x = x + extra_per_block_pos_emb + for block in self.blocks: + x = block( + x, + emb_B_D, + crossattn_emb, + crossattn_mask, + rope_emb_L_1_1_D=rope_emb_L_1_1_D, + adaln_lora_B_3D=adaln_lora_B_3D, + ) + return x diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/position_embedding.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/position_embedding.py new file mode 100644 index 00000000..f260bf4d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/position_embedding.py @@ -0,0 +1,449 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import numpy as np +import torch +from cosmos1.models.diffusion.module.attention import normalize +from cosmos1.models.diffusion.module.timm import trunc_normal_ +from einops import rearrange, repeat +from torch import nn + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +class VideoPositionEmb(nn.Module): + def forward(self, x_B_T_H_W_C: torch.Tensor, fps=Optional[torch.Tensor]) -> torch.Tensor: + """ + It delegates the embedding generation to generate_embeddings function. + """ + B_T_H_W_C = x_B_T_H_W_C.shape + embeddings = self.generate_embeddings(B_T_H_W_C, fps=fps) + + return embeddings + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]): + raise NotImplementedError + + +class VideoRopePosition3DEmb(VideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + head_dim: int, + len_h: int, + len_w: int, + len_t: int, + base_fps: int = 24, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + **kwargs, # used for compatibility with other positional embeddings; unused in this class + ): + del kwargs + super().__init__() + self.register_buffer("seq", torch.arange(max(len_h, len_w, len_t), dtype=torch.float)) + self.base_fps = base_fps + self.max_h = len_h + self.max_w = len_w + + dim = head_dim + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + self.register_buffer( + "dim_spatial_range", + torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().cuda() / dim_h, + persistent=False, + ) + self.register_buffer( + "dim_temporal_range", + torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().cuda() / dim_t, + persistent=False, + ) + + self.h_ntk_factor = h_extrapolation_ratio ** (dim_h / (dim_h - 2)) + self.w_ntk_factor = w_extrapolation_ratio ** (dim_w / (dim_w - 2)) + self.t_ntk_factor = t_extrapolation_ratio ** (dim_t / (dim_t - 2)) + + def generate_embeddings( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. + + Returns: + Not specified in the original code snippet. + """ + h_ntk_factor = h_ntk_factor if h_ntk_factor is not None else self.h_ntk_factor + w_ntk_factor = w_ntk_factor if w_ntk_factor is not None else self.w_ntk_factor + t_ntk_factor = t_ntk_factor if t_ntk_factor is not None else self.t_ntk_factor + + h_theta = 10000.0 * h_ntk_factor + w_theta = 10000.0 * w_ntk_factor + t_theta = 10000.0 * t_ntk_factor + + h_spatial_freqs = 1.0 / (h_theta**self.dim_spatial_range) + w_spatial_freqs = 1.0 / (w_theta**self.dim_spatial_range) + temporal_freqs = 1.0 / (t_theta**self.dim_temporal_range) + + B, T, H, W, _ = B_T_H_W_C + uniform_fps = (fps is None) or (fps.min() == fps.max()) + assert uniform_fps or B == 1 or T == 1, ( + "For video batch, batch size should be 1 for non-uniform fps. For image batch, T should be 1" + ) + assert H <= self.max_h and W <= self.max_w, ( + f"Input dimensions (H={H}, W={W}) exceed the maximum dimensions (max_h={self.max_h}, max_w={self.max_w})" + ) + half_emb_h = torch.outer(self.seq[:H], h_spatial_freqs) + half_emb_w = torch.outer(self.seq[:W], w_spatial_freqs) + + # apply sequence scaling in temporal dimension + if fps is None: # image case + assert T == 1, "T should be 1 for image batch." + half_emb_t = torch.outer(self.seq[:T], temporal_freqs) + else: + half_emb_t = torch.outer(self.seq[:T] / fps[:1] * self.base_fps, temporal_freqs) + + em_T_H_W_D = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + + return rearrange(em_T_H_W_D, "t h w d -> (t h w) 1 1 d").float() + + +class LearnablePosEmbAxis(VideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + interpolation: str, + model_channels: int, + len_h: int, + len_w: int, + len_t: int, + **kwargs, + ): + """ + Args: + interpolation (str): we curretly only support "crop", ideally when we need extrapolation capacity, we should adjust frequency or other more advanced methods. they are not implemented yet. + """ + del kwargs # unused + super().__init__() + self.interpolation = interpolation + assert self.interpolation in ["crop"], f"Unknown interpolation method {self.interpolation}" + + self.pos_emb_h = nn.Parameter(torch.zeros(len_h, model_channels)) + self.pos_emb_w = nn.Parameter(torch.zeros(len_w, model_channels)) + self.pos_emb_t = nn.Parameter(torch.zeros(len_t, model_channels)) + + trunc_normal_(self.pos_emb_h, std=0.02) + trunc_normal_(self.pos_emb_w, std=0.02) + trunc_normal_(self.pos_emb_t, std=0.02) + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]) -> torch.Tensor: + B, T, H, W, _ = B_T_H_W_C + if self.interpolation == "crop": + emb_h_H = self.pos_emb_h[:H] + emb_w_W = self.pos_emb_w[:W] + emb_t_T = self.pos_emb_t[:T] + emb = ( + repeat(emb_t_T, "t d-> b t h w d", b=B, h=H, w=W) + + repeat(emb_h_H, "h d-> b t h w d", b=B, t=T, w=W) + + repeat(emb_w_W, "w d-> b t h w d", b=B, t=T, h=H) + ) + assert list(emb.shape)[:4] == [B, T, H, W], f"bad shape: {list(emb.shape)[:4]} != {B, T, H, W}" + else: + raise ValueError(f"Unknown interpolation method {self.interpolation}") + + return normalize(emb, dim=-1, eps=1e-6) + + +class MultiCameraVideoPositionEmb(nn.Module): + def __init__( + self, + ): + super().__init__() + + def forward(self, x_B_T_H_W_C: torch.Tensor, fps=Optional[torch.Tensor]) -> torch.Tensor: + """ + With CP, the function assume that the input tensor is already split. It delegates the embedding generation to generate_embeddings function. + """ + B_T_H_W_C = x_B_T_H_W_C.shape + embeddings = self.generate_embeddings(B_T_H_W_C, fps=fps) + if isinstance(self, MultiCameraVideoRopePosition3DEmb): + embeddings = rearrange(embeddings, "t h w d -> (t h w) 1 1 d").float() + + return embeddings + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]): + raise NotImplementedError + + +class MultiCameraVideoRopePosition3DEmb(MultiCameraVideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + head_dim: int, + len_h: int, + len_w: int, + len_t: int, + base_fps: int = 24, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + n_cameras: int = 4, + **kwargs, # used for compatibility with other positional embeddings; unused in this class + ): + del kwargs + super().__init__() + self.register_buffer("seq", torch.arange(max(len_h, len_w, len_t), dtype=torch.float)) + self.base_fps = base_fps + self.max_h = len_h + self.max_w = len_w + self.n_cameras = n_cameras + dim = head_dim + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + self.register_buffer( + "dim_spatial_range", + torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().cuda() / dim_h, + persistent=False, + ) + self.register_buffer( + "dim_temporal_range", + torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().cuda() / dim_t, + persistent=False, + ) + + self.h_ntk_factor = h_extrapolation_ratio ** (dim_h / (dim_h - 2)) + self.w_ntk_factor = w_extrapolation_ratio ** (dim_w / (dim_w - 2)) + self.t_ntk_factor = t_extrapolation_ratio ** (dim_t / (dim_t - 2)) + + def generate_embedding_for_batch( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. Defaults to None. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. Defaults to None. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. Defaults to None. + + Returns: + Not specified in the original code snippet. + """ + h_ntk_factor = h_ntk_factor if h_ntk_factor is not None else self.h_ntk_factor + w_ntk_factor = w_ntk_factor if w_ntk_factor is not None else self.w_ntk_factor + t_ntk_factor = t_ntk_factor if t_ntk_factor is not None else self.t_ntk_factor + + h_theta = 10000.0 * h_ntk_factor + w_theta = 10000.0 * w_ntk_factor + t_theta = 10000.0 * t_ntk_factor + + h_spatial_freqs = 1.0 / (h_theta**self.dim_spatial_range) + w_spatial_freqs = 1.0 / (w_theta**self.dim_spatial_range) + temporal_freqs = 1.0 / (t_theta**self.dim_temporal_range) + + B, T, H, W, _ = B_T_H_W_C + uniform_fps = (fps is None) or (fps.min() == fps.max()) + assert uniform_fps # only support uniform fps now + + assert uniform_fps or B == 1 or T == 1, ( + "For video batch, batch size should be 1 for non-uniform fps. For image batch, T should be 1" + ) + assert H <= self.max_h and W <= self.max_w, ( + f"Input dimensions (H={H}, W={W}) exceed the maximum dimensions (max_h={self.max_h}, max_w={self.max_w}) configured for positional embedding. Please adjust the input size or increase the maximum dimensions in the model configuration." + ) + half_emb_h = torch.outer(self.seq[:H], h_spatial_freqs) + half_emb_w = torch.outer(self.seq[:W], w_spatial_freqs) + + # apply sequence scaling in temporal dimension + if fps is None: # image case + assert T == 1, "T should be 1 for image batch." + half_emb_t = torch.outer(self.seq[:T], temporal_freqs) + else: + half_emb_t = torch.outer(self.seq[:T] / fps[:1] * self.base_fps, temporal_freqs) + + em_T_H_W_D = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + + return em_T_H_W_D + + def generate_embeddings( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. The camera view dimension is merged in the T dimension + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time * Views, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. Defaults to None. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. Defaults to None. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. Defaults to None. + + Returns: + Not specified in the original code snippet. + """ + + B, T, H, W, C = B_T_H_W_C + + single_camera_B_T_H_W_C = (B, T // self.n_cameras, H, W, C) + em_T_H_W_D = torch.cat( + [ + self.generate_embedding_for_batch( + single_camera_B_T_H_W_C, + fps=fps, + h_ntk_factor=h_ntk_factor, + w_ntk_factor=w_ntk_factor, + t_ntk_factor=t_ntk_factor, + ) + for item in range(self.n_cameras) + ], + dim=0, + ) + return em_T_H_W_D + + +class MultiCameraSinCosPosEmbAxis(MultiCameraVideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + interpolation: str, + model_channels: int, + len_h: int, + len_w: int, + len_t: int, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + n_cameras: int = 4, + **kwargs, + ): + """ + Args: + interpolation (str): we curretly only support "crop", ideally when we need extrapolation capacity, we should adjust frequency or other more advanced methods. they are not implemented yet. + """ + del kwargs # unused + self.n_cameras = n_cameras + super().__init__() + self.interpolation = interpolation + assert self.interpolation in ["crop"], f"Unknown interpolation method {self.interpolation}" + + dim = model_channels + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + + # rescale pos id is equivalent to rescale frequency + emb_h = get_1d_sincos_pos_embed_from_grid(dim_h, pos=np.arange(len_h) * 1.0 / h_extrapolation_ratio) + emb_w = get_1d_sincos_pos_embed_from_grid(dim_w, pos=np.arange(len_w) * 1.0 / w_extrapolation_ratio) + emb_t = get_1d_sincos_pos_embed_from_grid(dim_t, pos=np.arange(len_t) * 1.0 / t_extrapolation_ratio) + + self.register_buffer("pos_emb_h", torch.from_numpy(emb_h).float(), persistent=False) + self.register_buffer("pos_emb_w", torch.from_numpy(emb_w).float(), persistent=False) + self.register_buffer("pos_emb_t", torch.from_numpy(emb_t).float(), persistent=False) + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]) -> torch.Tensor: + B, T, H, W, C = B_T_H_W_C + + single_camera_T = T // self.n_cameras + + if self.interpolation == "crop": + emb_h_H = self.pos_emb_h[:H] + emb_w_W = self.pos_emb_w[:W] + emb_t_T = self.pos_emb_t[:single_camera_T] + emb = torch.cat( + [ + torch.cat( + [ + repeat(emb_t_T, "t d-> b t h w d", b=B, h=H, w=W), + repeat(emb_h_H, "h d-> b t h w d", b=B, t=single_camera_T, w=W), + repeat(emb_w_W, "w d-> b t h w d", b=B, t=single_camera_T, h=H), + ], + dim=-1, + ) + for _ in range(self.n_cameras) + ], + 1, + ) + assert list(emb.shape)[:4] == [B, T, H, W], f"bad shape: {list(emb.shape)[:4]} != {B, T, H, W}" + return emb + + raise ValueError(f"Unknown interpolation method {self.interpolation}") diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/pretrained_vae.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/pretrained_vae.py new file mode 100644 index 00000000..27593f76 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/pretrained_vae.py @@ -0,0 +1,610 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from abc import ABC, abstractmethod + +import torch +from einops import rearrange +from torch.nn.modules import Module + + +class BaseVAE(torch.nn.Module, ABC): + """ + Abstract base class for a Variational Autoencoder (VAE). + + All subclasses should implement the methods to define the behavior for encoding + and decoding, along with specifying the latent channel size. + """ + + def __init__(self, channel: int = 3, name: str = "vae"): + super().__init__() + self.channel = channel + self.name = name + + @property + def latent_ch(self) -> int: + """ + Returns the number of latent channels in the VAE. + """ + return self.channel + + @abstractmethod + def encode(self, state: torch.Tensor) -> torch.Tensor: + """ + Encodes the input tensor into a latent representation. + + Args: + - state (torch.Tensor): The input tensor to encode. + + Returns: + - torch.Tensor: The encoded latent tensor. + """ + pass + + @abstractmethod + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decodes the latent representation back to the original space. + + Args: + - latent (torch.Tensor): The latent tensor to decode. + + Returns: + - torch.Tensor: The decoded tensor. + """ + pass + + @property + def spatial_compression_factor(self) -> int: + """ + Returns the spatial reduction factor for the VAE. + """ + raise NotImplementedError("The spatial_compression_factor property must be implemented in the derived class.") + + +class BasePretrainedImageVAE(BaseVAE): + """ + A base class for pretrained Variational Autoencoder (VAE) that loads mean and standard deviation values + from a remote store, handles data type conversions, and normalization + using provided mean and standard deviation values for latent space representation. + Derived classes should load pre-trained encoder and decoder components from a remote store + + Attributes: + latent_mean (Tensor): The mean used for normalizing the latent representation. + latent_std (Tensor): The standard deviation used for normalizing the latent representation. + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + mean_std_fp (str): File path to the pickle file containing mean and std of the latent space. + latent_ch (int, optional): Number of latent channels (default is 16). + is_image (bool, optional): Flag to indicate whether the output is an image (default is True). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + """ + + def __init__( + self, + name: str, + latent_ch: int = 16, + is_image: bool = True, + is_bf16: bool = True, + ) -> None: + super().__init__(latent_ch, name) + dtype = torch.bfloat16 if is_bf16 else torch.float32 + self.dtype = dtype + self.is_image = is_image + self.name = name + + def register_mean_std(self, vae_dir: str) -> None: + latent_mean, latent_std = torch.load(os.path.join(vae_dir, "image_mean_std.pt"), weights_only=True) + + target_shape = [1, self.latent_ch, 1, 1] if self.is_image else [1, self.latent_ch, 1, 1, 1] + + self.register_buffer( + "latent_mean", + latent_mean.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + self.register_buffer( + "latent_std", + latent_std.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + """ + Encode the input state to latent space; also handle the dtype conversion, mean and std scaling + """ + in_dtype = state.dtype + latent_mean = self.latent_mean.to(in_dtype) + latent_std = self.latent_std.to(in_dtype) + encoded_state = self.encoder(state.to(self.dtype)) + if isinstance(encoded_state, torch.Tensor): + pass + elif isinstance(encoded_state, tuple): + assert isinstance(encoded_state[0], torch.Tensor) + encoded_state = encoded_state[0] + else: + raise ValueError("Invalid type of encoded state") + return (encoded_state.to(in_dtype) - latent_mean) / latent_std + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decode the input latent to state; also handle the dtype conversion, mean and std scaling + """ + in_dtype = latent.dtype + latent = latent * self.latent_std.to(in_dtype) + self.latent_mean.to(in_dtype) + return self.decoder(latent.to(self.dtype)).to(in_dtype) + + def reset_dtype(self, *args, **kwargs): + """ + Resets the data type of the encoder and decoder to the model's default data type. + + Args: + *args, **kwargs: Unused, present to allow flexibility in method calls. + """ + del args, kwargs + self.decoder.to(self.dtype) + self.encoder.to(self.dtype) + + +class JITVAE(BasePretrainedImageVAE): + """ + A JIT compiled Variational Autoencoder (VAE) that loads pre-trained encoder + and decoder components from a remote store, handles data type conversions, and normalization + using provided mean and standard deviation values for latent space representation. + + Attributes: + encoder (Module): The JIT compiled encoder loaded from storage. + decoder (Module): The JIT compiled decoder loaded from storage. + latent_mean (Tensor): The mean used for normalizing the latent representation. + latent_std (Tensor): The standard deviation used for normalizing the latent representation. + dtype (dtype): Data type for model tensors, determined by whether bf16 is enabled. + + Args: + name (str): Name of the model, used for differentiating cache file paths. + latent_ch (int, optional): Number of latent channels (default is 16). + is_image (bool, optional): Flag to indicate whether the output is an image (default is True). + is_bf16 (bool, optional): Flag to use Brain Floating Point 16-bit data type (default is True). + """ + + def __init__( + self, + name: str, + latent_ch: int = 16, + is_image: bool = True, + is_bf16: bool = True, + ): + super().__init__(name, latent_ch, is_image, is_bf16) + + def load_encoder(self, vae_dir: str) -> None: + """ + Load the encoder from the remote store. + """ + self.encoder = torch.load(os.path.join(vae_dir, "encoder.jit"), weights_only=True) + + self.encoder.eval() + for param in self.encoder.parameters(): + param.requires_grad = False + self.encoder.to(self.dtype) + + def load_decoder(self, vae_dir: str) -> None: + """ + Load the decoder from the remote store. + """ + self.decoder = torch.load(os.path.join(vae_dir, "decoder.jit"), weights_only=True) + + self.decoder.eval() + for param in self.decoder.parameters(): + param.requires_grad = False + self.decoder.to(self.dtype) + + +class BaseVAE(torch.nn.Module, ABC): + """ + Abstract base class for a Variational Autoencoder (VAE). + + All subclasses should implement the methods to define the behavior for encoding + and decoding, along with specifying the latent channel size. + """ + + def __init__(self, channel: int = 3, name: str = "vae"): + super().__init__() + self.channel = channel + self.name = name + + @property + def latent_ch(self) -> int: + """ + Returns the number of latent channels in the VAE. + """ + return self.channel + + @abstractmethod + def encode(self, state: torch.Tensor) -> torch.Tensor: + """ + Encodes the input tensor into a latent representation. + + Args: + - state (torch.Tensor): The input tensor to encode. + + Returns: + - torch.Tensor: The encoded latent tensor. + """ + pass + + @abstractmethod + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decodes the latent representation back to the original space. + + Args: + - latent (torch.Tensor): The latent tensor to decode. + + Returns: + - torch.Tensor: The decoded tensor. + """ + pass + + @property + def spatial_compression_factor(self) -> int: + """ + Returns the spatial reduction factor for the VAE. + """ + raise NotImplementedError("The spatial_compression_factor property must be implemented in the derived class.") + + +class VideoTokenizerInterface(ABC): + @abstractmethod + def encode(self, state: torch.Tensor) -> torch.Tensor: + pass + + @abstractmethod + def decode(self, latent: torch.Tensor) -> torch.Tensor: + pass + + @abstractmethod + def get_latent_num_frames(self, num_pixel_frames: int) -> int: + pass + + @abstractmethod + def get_pixel_num_frames(self, num_latent_frames: int) -> int: + pass + + @property + @abstractmethod + def spatial_compression_factor(self): + pass + + @property + @abstractmethod + def temporal_compression_factor(self): + pass + + @property + @abstractmethod + def spatial_resolution(self): + pass + + @property + @abstractmethod + def pixel_chunk_duration(self): + pass + + @property + @abstractmethod + def latent_chunk_duration(self): + pass + + +class BasePretrainedVideoTokenizer(ABC): + """ + Base class for a pretrained video tokenizer that handles chunking of video data for efficient processing. + + Args: + pixel_chunk_duration (int): The duration (in number of frames) of each chunk of video data at the pixel level. + temporal_compress_factor (int): The factor by which the video data is temporally compressed during processing. + max_enc_batch_size (int): The maximum batch size to process in one go during encoding to avoid memory overflow. + max_dec_batch_size (int): The maximum batch size to process in one go during decoding to avoid memory overflow. + + The class introduces parameters for managing temporal chunks (`pixel_chunk_duration` and `temporal_compress_factor`) + which define how video data is subdivided and compressed during the encoding and decoding processes. The + `max_enc_batch_size` and `max_dec_batch_size` parameters allow processing in smaller batches to handle memory + constraints. + """ + + def __init__( + self, + pixel_chunk_duration: int = 17, + temporal_compress_factor: int = 8, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + ): + self._pixel_chunk_duration = pixel_chunk_duration + self._temporal_compress_factor = temporal_compress_factor + self.max_enc_batch_size = max_enc_batch_size + self.max_dec_batch_size = max_dec_batch_size + + def register_mean_std(self, vae_dir: str) -> None: + latent_mean, latent_std = torch.load(os.path.join(vae_dir, "mean_std.pt"), weights_only=True) + + latent_mean = latent_mean.view(self.latent_ch, -1)[:, : self.latent_chunk_duration] + latent_std = latent_std.view(self.latent_ch, -1)[:, : self.latent_chunk_duration] + + target_shape = [1, self.latent_ch, self.latent_chunk_duration, 1, 1] + + self.register_buffer( + "latent_mean", + latent_mean.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + self.register_buffer( + "latent_std", + latent_std.to(self.dtype).reshape(*target_shape), + persistent=False, + ) + + def transform_encode_state_shape(self, state: torch.Tensor) -> torch.Tensor: + """ + Rearranges the input state tensor to the required shape for encoding video data. Mainly for chunk based encoding + """ + B, C, T, H, W = state.shape + assert T % self.pixel_chunk_duration == 0, ( + f"Temporal dimension {T} is not divisible by chunk_length {self.pixel_chunk_duration}" + ) + return rearrange(state, "b c (n t) h w -> (b n) c t h w", t=self.pixel_chunk_duration) + + def transform_decode_state_shape(self, latent: torch.Tensor) -> torch.Tensor: + B, _, T, _, _ = latent.shape + assert T % self.latent_chunk_duration == 0, ( + f"Temporal dimension {T} is not divisible by chunk_length {self.latent_chunk_duration}" + ) + return rearrange(latent, "b c (n t) h w -> (b n) c t h w", t=self.latent_chunk_duration) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + origin_T = None + if self._temporal_compress_factor == 1: + _, _, origin_T, _, _ = state.shape + state = rearrange(state, "b c t h w -> (b t) c 1 h w") + B, C, T, H, W = state.shape + state = self.transform_encode_state_shape(state) + # use max_enc_batch_size to avoid OOM + if state.shape[0] > self.max_enc_batch_size: + latent = [] + for i in range(0, state.shape[0], self.max_enc_batch_size): + latent.append(super().encode(state[i : i + self.max_enc_batch_size])) + latent = torch.cat(latent, dim=0) + else: + latent = super().encode(state) + + latent = rearrange(latent, "(b n) c t h w -> b c (n t) h w", b=B) + if self._temporal_compress_factor == 1: + latent = rearrange(latent, "(b t) c 1 h w -> b c t h w", t=origin_T) + return latent + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """ + Decodes a batch of latent representations into video frames by applying temporal chunking. Similar to encode, + it handles video data by processing smaller temporal chunks to reconstruct the original video dimensions. + + It can also decode single frame image data. + + Args: + latent (torch.Tensor): The latent space tensor containing encoded video data. + + Returns: + torch.Tensor: The decoded video tensor reconstructed from latent space. + """ + origin_T = None + if self._temporal_compress_factor == 1: + _, _, origin_T, _, _ = latent.shape + latent = rearrange(latent, "b c t h w -> (b t) c 1 h w") + B, _, T, _, _ = latent.shape + latent = self.transform_decode_state_shape(latent) + # use max_enc_batch_size to avoid OOM + if latent.shape[0] > self.max_dec_batch_size: + state = [] + for i in range(0, latent.shape[0], self.max_dec_batch_size): + state.append(super().decode(latent[i : i + self.max_dec_batch_size])) + state = torch.cat(state, dim=0) + else: + state = super().decode(latent) + assert state.shape[2] == self.pixel_chunk_duration + state = rearrange(state, "(b n) c t h w -> b c (n t) h w", b=B) + if self._temporal_compress_factor == 1: + return rearrange(state, "(b t) c 1 h w -> b c t h w", t=origin_T) + return state + + @property + def pixel_chunk_duration(self) -> int: + return self._pixel_chunk_duration + + @property + def latent_chunk_duration(self) -> int: + # return self._latent_chunk_duration + assert (self.pixel_chunk_duration - 1) % self.temporal_compression_factor == 0, ( + f"Pixel chunk duration {self.pixel_chunk_duration} is not divisible by latent chunk duration " + f"{self.latent_chunk_duration}" + ) + return (self.pixel_chunk_duration - 1) // self.temporal_compression_factor + 1 + + @property + def temporal_compression_factor(self): + return self._temporal_compress_factor + + def get_latent_num_frames(self, num_pixel_frames: int) -> int: + if num_pixel_frames == 1: + return 1 + assert num_pixel_frames % self.pixel_chunk_duration == 0, ( + f"Temporal dimension {num_pixel_frames} is not divisible by chunk_length {self.pixel_chunk_duration}" + ) + return num_pixel_frames // self.pixel_chunk_duration * self.latent_chunk_duration + + def get_pixel_num_frames(self, num_latent_frames: int) -> int: + if num_latent_frames == 1: + return 1 + assert num_latent_frames % self.latent_chunk_duration == 0, ( + f"Temporal dimension {num_latent_frames} is not divisible by chunk_length {self.latent_chunk_duration}" + ) + return num_latent_frames // self.latent_chunk_duration * self.pixel_chunk_duration + + +class VideoJITTokenizer(BasePretrainedVideoTokenizer, JITVAE, VideoTokenizerInterface): + """ + Instance of BasePretrainedVideoVAE that loads encoder and decoder from JIT scripted module file + """ + + def __init__( + self, + name: str, + latent_ch: int = 16, + is_bf16: bool = True, + spatial_compression_factor: int = 16, + temporal_compression_factor: int = 8, + pixel_chunk_duration: int = 17, + max_enc_batch_size: int = 8, + max_dec_batch_size: int = 4, + spatial_resolution: str = "720", + ): + super().__init__( + pixel_chunk_duration, + temporal_compression_factor, + max_enc_batch_size, + max_dec_batch_size, + ) + super(BasePretrainedVideoTokenizer, self).__init__( + name, + latent_ch, + False, + is_bf16, + ) + + self._spatial_compression_factor = spatial_compression_factor + self._spatial_resolution = spatial_resolution + + @property + def spatial_compression_factor(self): + return self._spatial_compression_factor + + @property + def spatial_resolution(self) -> str: + return self._spatial_resolution + + +class JointImageVideoTokenizer(BaseVAE, VideoTokenizerInterface): + def __init__( + self, + image_vae: torch.nn.Module, + video_vae: torch.nn.Module, + name: str, + latent_ch: int = 16, + squeeze_for_image: bool = True, + ): + super().__init__(latent_ch, name) + self.image_vae = image_vae + self.video_vae = video_vae + self.squeeze_for_image = squeeze_for_image + + def encode_image(self, state: torch.Tensor) -> torch.Tensor: + if self.squeeze_for_image: + return self.image_vae.encode(state.squeeze(2)).unsqueeze(2) + return self.image_vae.encode(state) + + def decode_image(self, latent: torch.Tensor) -> torch.Tensor: + if self.squeeze_for_image: + return self.image_vae.decode(latent.squeeze(2)).unsqueeze(2) + return self.image_vae.decode(latent) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + B, C, T, H, W = state.shape + if T == 1: + return self.encode_image(state) + + return self.video_vae.encode(state) + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + B, C, T, H, W = latent.shape + if T == 1: + return self.decode_image(latent) + return self.video_vae.decode(latent) + + def reset_dtype(self, *args, **kwargs): + """ + Resets the data type of the encoder and decoder to the model's default data type. + + Args: + *args, **kwargs: Unused, present to allow flexibility in method calls. + """ + del args, kwargs + self.video_vae.reset_dtype() + + def get_latent_num_frames(self, num_pixel_frames: int) -> int: + if num_pixel_frames == 1: + return 1 + return self.video_vae.get_latent_num_frames(num_pixel_frames) + + def get_pixel_num_frames(self, num_latent_frames: int) -> int: + if num_latent_frames == 1: + return 1 + return self.video_vae.get_pixel_num_frames(num_latent_frames) + + @property + def spatial_compression_factor(self): + return self.video_vae.spatial_compression_factor + + @property + def temporal_compression_factor(self): + return self.video_vae.temporal_compression_factor + + @property + def spatial_resolution(self) -> str: + return self.video_vae.spatial_resolution + + @property + def pixel_chunk_duration(self) -> int: + return self.video_vae.pixel_chunk_duration + + @property + def latent_chunk_duration(self) -> int: + return self.video_vae.latent_chunk_duration + + +class JointImageVideoSharedJITTokenizer(JointImageVideoTokenizer): + """ + First version of the ImageVideoVAE trained with Fitsum. + We have to use seperate mean and std for image and video due to non-causal nature of the model. + """ + + def __init__(self, image_vae: Module, video_vae: Module, name: str, latent_ch: int = 16): + super().__init__(image_vae, video_vae, name, latent_ch, squeeze_for_image=False) + assert isinstance(image_vae, JITVAE) + assert isinstance(video_vae, VideoJITTokenizer), ( + f"video_vae should be an instance of VideoJITVAE, got {type(video_vae)}" + ) + # a hack to make the image_vae and video_vae share the same encoder and decoder + + def load_weights(self, vae_dir: str): + self.video_vae.register_mean_std(vae_dir) + + self.video_vae.load_decoder(vae_dir) + self.video_vae.load_encoder(vae_dir) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/timm.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/timm.py new file mode 100644 index 00000000..cb5ebf54 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/module/timm.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import warnings + +import torch + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +# flake8: noqa: E741 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/__init__.py new file mode 100644 index 00000000..b11bcc2d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/download_diffusion_nemo.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/download_diffusion_nemo.py new file mode 100644 index 00000000..26b29f52 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/download_diffusion_nemo.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +from huggingface_hub import snapshot_download + + +def download_diffusion_nemo(): + """ + Downloads all Cosmos Diffusion NeMo assets to HF_HOME directory. + Make sure to set HF_HOME to your desired path before running this function. + """ + snapshot_download("nvidia/Cosmos-1.0-Guardrail") + snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Text2World", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Diffusion-14B-Text2World", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Diffusion-14B-Video2World", allow_patterns=["nemo/*"]) + snapshot_download("nvidia/Cosmos-1.0-Prompt-Upsampler-12B-Text2World") + snapshot_download("google-t5/t5-11b", ignore_patterns=["*.h5"]) + + +def main(): + # Check if HF_HOME is set + hf_home = os.environ.get("HF_HOME") + if not hf_home: + raise EnvironmentError( + "The HF_HOME environment variable is not set. " + "Please set it to your desired path before running this script." + ) + + # Download Cosmos Diffusion NeMo checkpoints + download_diffusion_nemo() + + +if __name__ == "__main__": + main() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/README.md new file mode 100644 index 00000000..d52cb0c4 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/README.md @@ -0,0 +1,413 @@ +# Cosmos Diffusion-based World Foundation Models: NeMo Framework User Guide + +Learn how to [run inference](#inference) with Cosmos Diffusion-based World Foundation Models (WFMs) using the [NVIDIA NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html) for your custom Physical AI tasks by following this guide. + +## Model Support Matrix + +The NeMo Framework supports the following Cosmos Diffusion models. Review the available models and their compute requirements for post-tuning and inference to determine the best model for your use case. + +| Model Name | Model Status | Compute Requirements for Inference | Multi-GPU Support +|----------------------------------------------|------------------|------------------------------------------|---------| +| Cosmos-1.0-Diffusion-7B-Text2World | **Supported** | 1 NVIDIA GPU* | **Supported** | +| Cosmos-1.0-Diffusion-14B-Text2World | **Supported** | 1 NVIDIA GPU* | **Supported** | +| Cosmos-1.0-Diffusion-7B-Video2World | **Supported** | 1 NVIDIA GPU* | **Supported** | +| Cosmos-1.0-Diffusion-14B-Video2WorldB | **Supported** | 1 NVIDIA GPU* | **Supported** | + + +**\*** `H100-80GB` or `A100-80GB` GPUs are recommended. + +## Post-Trained Model Inference Support Matrix + +Cosmos Diffusion-based WFMs can also be post-trained for a variety of Physical AI tasks and used for inference. Review the following table for a list of available Physical AI post-training tasks: + +| Post-training Task | Inference Support Status | +|-------------------------|--------------------| +| General post-training | **Supported** | +| Instruction control | **Supported** | +| Action control | **Supported** | +| Camera control | **Supported** | +| Multi-view generation | **Supported** | +| Multi-view generation with vehicle trajectory control | **Supported** | + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the minimum compute required to run the model(s), as listed in the model support matrix. + - **Containerization Platform**: We recommend using Docker with NVIDIA Container Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Clone the Cosmos Repository + +```bash +git clone git@github.com:NVIDIA/Cosmos.git +``` + +### 3. Start the Container + +The [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos Diffusion models. + +Run the following command to download and start the container: +```bash +docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidian/nemo:cosmos.1.0.2 bash +``` + +### 4. Download Checkpoints + +To help you get started, we've provided a [download script](../download_diffusion_nemo.py) to get the Cosmos Diffusion Text2World and Video2World checkpoints from Hugging Face. These checkpoints are in the NeMo distributed checkpoint format required to run post-training and inference with NeMo Framework. + +1. Set the following environment variables: + + ```bash + # You must set HF_HOME before running this script. + export HF_TOKEN="" + export HF_HOME="" + ``` + +2. Run the following command to download the models: + + ```bash + cd /workspace/Cosmos + python cosmos1/models/diffusion/nemo/download_diffusion_nemo.py + ``` + +## Run Inference + +Running inference with Cosmos Diffusion Text2World models lets you generate a video conditioned on a text prompt. With the Video2World models, you can generate a video conditioned on a text prompt as well as on an image or video. Note that when supplying an image or video for conditioning the following requirements must be met: + +- **Video**: The video must be less than 9 frames long +- **Image**: The image must be either PNG or JPEG format and have one of the following extensions: `.png`, `.jpg`, or `.jpeg` + +Our inference script enables accelerated world generation with context parallel. We use context parallelism to split the diffusion process across multiple GPUs, providing near-linear scaling efficiency. Our diffusion pipeline also allows the user to set a variety of hyperparameters including the random seed, classifier-free guidance scale, negative prompt, video resolution, and video fps. + +General post-training is essentially a continuation of pre-training. To perform inference with models that have been post-trained with general post-training, you can set the `subject_name` parameter to the subject the model was post-trained on. The `prompt` and `conditioned_image_or_video_path` parameters are then used to provide the setting and describe the events in the generated world. The final prompt will be "A video of sks `{subject_name}`. `{prompt}`". We can also use [inference/general.py](./general.py) or [inference/video2world.py](./video2world.py) to perform inference on the base models since the model architectures are the same as the general post-trained models. + +We also provide the option to upsample the `prompt` and make it more detailed. This can improve the quality of the generated world. Note that for Video2World generation, currently the LLM only looks at your text prompt to upsample the initial prompt, and it does not consider your input image/video for prompt upsampling. We will add text + image processing for prompt upsampling in the near future. + +### Run the Inference Script with Base Models + +#### Text2World + +Complete the following steps to generate a new output video of a robot cooking. + +1. Set the following environment variables: + + ```bash + # HuggingFace Cache to save T5 text encoder, video tokenizer, prompt upsampler, and guardrails weights. + export HF_TOKEN="" + export HF_HOME="" + + # Number of GPU devices available for inference. Supports up to 8 GPUs for accelerated inference. + export NUM_DEVICES=1 + export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $((NUM_DEVICES - 1))) + + # Prompt describing world scene and actions taken by subject (if provided). + export PROMPT="The teal robot is cooking food in a kitchen. Steam rises from a simmering pot as the robot chops vegetables on a worn wooden cutting board. Copper pans hang from an overhead rack, catching glints of afternoon light, while a well-loved cast iron skillet sits on the stovetop next to scattered measuring spoons and a half-empty bottle of olive oil." + ``` + +2. Run the following command: + + ```bash + NVTE_FUSED_ATTN=0 \ + torchrun --nproc_per_node=$NUM_DEVICES cosmos1/models/diffusion/nemo/inference/general.py \ + --model Cosmos-1.0-Diffusion-7B-Text2World \ + --cp_size $NUM_DEVICES \ + --num_devices $NUM_DEVICES \ + --video_save_path "Cosmos-1.0-Diffusion-7B-Text2World.mp4" \ + --guidance 7 \ + --seed 1 \ + --prompt "$PROMPT" \ + --enable_prompt_upsampler + ``` + +#### Video2World + +Complete the following steps to generate a new output video conditioned on an input video and a text prompt using the Video2World models. + +1. Set the following environment variables: + + ```bash + # HuggingFace Cache to save T5 text encoder, video tokenizer, prompt upsampler, and guardrails weights. + export HF_TOKEN="" + export HF_HOME="" + + # Number of GPU devices available for inference. Supports up to 8 GPUs for accelerated inference. + export NUM_DEVICES=1 + export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $((NUM_DEVICES - 1))) + + # Prompt describing world scene and actions taken by subject (if provided). + export PROMPT="" + export CONDITIONED_IMAGE_OR_VIDEO="" + ``` + +2. Run the following command: + + ```bash + NVTE_FUSED_ATTN=0 \ + torchrun --nproc_per_node=$NUM_DEVICES cosmos1/models/diffusion/nemo/inference/video2world.py \ + --model Cosmos-1.0-Diffusion-7B-Video2World \ + --cp_size $NUM_DEVICES \ + --num_devices $NUM_DEVICES \ + --video_save_path "Cosmos-1.0-Diffusion-7B-Video2World.mp4" \ + --guidance 7 \ + --seed 1 \ + --prompt "$PROMPT" \ + --conditioned_image_or_video_path "$CONDITIONED_IMAGE_OR_VIDEO" \ + --num_input_frames 9 \ + --enable_prompt_upsampler + ``` + +For Control2World, please use the following command instead: +```bash +export CTRL_MODEL_DIR=... +torchrun --nproc_per_node=$NUM_DEVICES cosmos1/models/diffusion/nemo/inference/control2world.py \ + --cosmos_assets_dir ./checkpoints \ + --conditioned_image_or_video_path $CONDITIONED_IMAGE_OR_VIDEO \ + --num_devices $NUM_DEVICES \ + --cp_size $NUM_DEVICES \ + --prompt "$PROMPT" \ + --control_weight 1 \ + --ctrl_model_dir $CTRL_MODEL_DIR +``` + +### Run the Inference Script with Post-trained Models + +Create a post-trained model first, by using the instructions [here](../post_training/README.md) +Then complete the following steps to generate a new output video from this model. + +#### Text2World + +1. Set the following environment variables: + + ```bash + # HuggingFace Cache to save T5 text encoder, video tokenizer, prompt upsampler, and guardrails weights. + export HF_TOKEN="" + export HF_HOME="" + + # Inference with post-trained model. Find post-trained model under nemo_experiments. Example path: + export NEMO_CHECKPOINT=nemo_experiments/cosmos_diffusion_7b_text2world_finetune/default/2024-12-17_01-28-03/checkpoints/epoch=39-step=199/weights + + # Number of GPU devices available for inference. Supports up to 8 GPUs for accelerated inference. + export NUM_DEVICES=1 + export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $((NUM_DEVICES - 1))) + + # Prompt describing world scene and actions taken by subject (if provided). + export PROMPT="The teal robot is cooking food in a kitchen. Steam rises from a simmering pot as the robot chops vegetables on a worn wooden cutting board. Copper pans hang from an overhead rack, catching glints of afternoon light, while a well-loved cast iron skillet sits on the stovetop next to scattered measuring spoons and a half-empty bottle of olive oil." + ``` + +2. Run the following command: + + ```bash + NVTE_FUSED_ATTN=0 \ + torchrun --nproc_per_node=8 cosmos1/models/diffusion/nemo/inference/general.py \ + --model Cosmos-1.0-Diffusion-7B-Text2World \ + --nemo_checkpoint "$NEMO_CHECKPOINT" \ + --cp_size $NUM_DEVICES \ + --num_devices $NUM_DEVICES \ + --video_save_path "Cosmos-1.0-Diffusion-7B-Text2World.mp4" \ + --guidance 7 \ + --seed 1 \ + --prompt "$PROMPT" \ + --subject_name "teal robot" \ + --enable_prompt_upsampler + ``` + +##### Example Output + +The following output is an example video generated from the post-trained model using [`general.py`](./inference/general.py): + + + +##### Configuration Options + +The following table details the parameters that can be modified for accelerated inference with NeMo. You can adjust these parameters to optimize performance based on your specific requirements. The model inference hyperparameters listed below have the same functionality as in [Cosmos Diffusion Common Parameters](cosmos1/models/diffusion/README.md#parameters). + + +| Parameter | Description | Default | +|--------------------------------|---------------------------------------------------------------------------------|---------| +| `--model` | Name of Cosmos Text2World Diffusion model to use for inference. | `Cosmos-1.0-Diffusion-7B-Text2World` | +| `--prompt` | Prompt which the sampled video is conditioned on. Tries to generate what is mentioned in the prompt. | *None* (user must provide) | +| `--negative_prompt` | Negative prompt for improved quality | "The video captures a series of frames showing ugly scenes..." | +| `--subject_name` | Name of the subject the model was post-trained on. This can be left empty for base model inference. | *None* | +| `--guidance` | A control mechanism that determines how closely the model follows specified conditions (prompt) during the generation process. We recommend starting with a guidance of 7 and increasing it later if necessary. | 7 | +| `--sampler` | Sampling method used for generation. Only supports **RES** sampler from [this paper](https://arxiv.org/pdf/2308.02157). | `RES` | +| `--video_save_path` | Location to save generated videos. | `Cosmos-1.0-Diffusion-7B-Text2World.mp4` | +| `--fps` | Frames-per-second of generated video. Cosmos Diffusion models generate videos at 24 FPS by default. | 24 | +| `--height` | Height of the generated video. Set to 704 pixels by default, which is the largest supported video height for Cosmos Diffusion. | 704 | +| `--width` | Width of the generated video. Set to 1280 pixels by default, which is the largest supported video width for Cosmos Diffusion. | 1280 | +| `--seed` | Random seed for generating initial noise sample. Changing this will create a different video for the same prompt. Keep the seed fixed to maintain deterministic video generations. | 1 | +| `--num_devices` | [1–8] Number of GPUs to use in parallel for inference. | 8 | +| `--cp_size` | [1–8] Number of context parallel ranks to spawn for parallelized inference. Must be equal to `num_devices`. | 8 | + +#### Video2World + +1. Set the following environment variables: + + ```bash + # HuggingFace Cache to save T5 text encoder, video tokenizer, prompt upsampler, and guardrails weights. + export HF_TOKEN="" + export HF_HOME="" + + # Inference with post-trained model. Find post-trained model under nemo_experiments. Example path: + export NEMO_CHECKPOINT=nemo_experiments/cosmos_diffusion_7b_video2world_finetune/default/2025-02-03_11-57-33/checkpoints/epoch=39-step=199/weights + + # Number of GPU devices available for inference. Supports up to 8 GPUs for accelerated inference. + export NUM_DEVICES=1 + export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $((NUM_DEVICES - 1))) + + export PROMPT="" + export CONDITIONED_IMAGE_OR_VIDEO="" + ``` + +2. Run the following command: + + ```bash + NVTE_FUSED_ATTN=0 \ + torchrun --nproc_per_node=8 cosmos1/models/diffusion/nemo/inference/video2world.py \ + --model Cosmos-1.0-Diffusion-7B-Video2World \ + --nemo_checkpoint "$NEMO_CHECKPOINT" \ + --cp_size $NUM_DEVICES \ + --num_devices $NUM_DEVICES \ + --video_save_path "Cosmos-1.0-Diffusion-7B-Video2World.mp4" \ + --guidance 7 \ + --seed 1 \ + --prompt "$PROMPT" \ + --conditioned_image_or_video_path "$CONDITIONED_IMAGE_OR_VIDEO" \ + --subject_name "teal robot" \ + --enable_prompt_upsampler + +##### Configuration Options + +The following table details the parameters that can be modified for accelerated inference with NeMo. You can adjust these parameters to optimize performance based on your specific requirements. The model inference hyperparameters listed below have the same functionality as in [Cosmos Diffusion Common Parameters](cosmos1/models/diffusion/README.md#parameters). + + +| Parameter | Description | Default | +|--------------------------------|---------------------------------------------------------------------------------|---------| +| `--model` | Name of Cosmos Video2World Diffusion model to use for inference. | `Cosmos-1.0-Diffusion-7B-Video2World` | +| `--prompt` | Prompt which the sampled video is conditioned on. Tries to generate what is mentioned in the prompt. | *None* (user must provide) | +| `--conditioned_image_or_video_path` | Input video used for conditioning generations. | *None* (user must provide) | +| `--negative_prompt` | Negative prompt for improved quality | "The video captures a series of frames showing ugly scenes..." | +| `--subject_name` | Name of the subject the model was post-trained on. This can be left empty for base model inference. | *None* | +| `--guidance` | A control mechanism that determines how closely the model follows specified conditions (prompt) during the generation process. We recommend starting with a guidance of 7 and increasing it later if necessary. | 7 | +| `--sampler` | Sampling method used for generation. Only supports **RES** sampler from [this paper](https://arxiv.org/pdf/2308.02157). | `RES` | +| `--video_save_path` | Location to save generated videos. | `Cosmos-1.0-Diffusion-7B-Video2World.mp4` | +| `--fps` | Frames-per-second of generated video. Cosmos Diffusion models generate videos at 24 FPS by default. | 24 | +| `--height` | Height of the generated video. Set to 704 pixels by default, which is the largest supported video height for Cosmos Diffusion. | 704 | +| `--width` | Width of the generated video. Set to 1280 pixels by default, which is the largest supported video width for Cosmos Diffusion. | 1280 | +| `--seed` | Random seed for generating initial noise sample. Changing this will create a different video for the same prompt. Keep the seed fixed to maintain deterministic video generations. | 1 | +| `--num_devices` | [1–8] Number of GPUs to use in parallel for inference. | 8 | +| `--cp_size` | [1–8] Number of context parallel ranks to spawn for parallelized inference. Must be equal to `num_devices`. | 8 | + + +Generated videos are saved at the location configured in the `SAVE_PATH` parameter. + +> **Tip**: +> For faster inference, you can remove the `--enable_prompt_upsampler` parameter, but this may degrade the generated result. + +> **Disclaimer**: +> The post-training example in this documentation is a demonstration of general post-training and not a guaranteed recipe for success. Post-training outcomes depend heavily on the quality and diversity of the dataset. To achieve good results, ensure your dataset is clean, well-structured, diverse, and properly labeled. Poorly prepared data can lead to issues like overfitting, bias, or poor performance. Carefully curate your dataset to reflect the desired use case for reliable results. + +## Run Inference Script on Action Control Models + +Inference on the tokenized dataset is supported. First generate a post-trained [action control checkpoint](../post_training/README.md). + +1. Run the following command to download and start the container: +```bash + docker run --ipc=host -it --gpus=all \ + -v :/opt/Cosmos \ + -v :/root/.cache/huggingface \ + -v :/checkpoint/ \ + nvcr.io/nvidian/nemo:cosmos.1.0.2 + ``` + +2. Set the following environment variables: +```bash + export HF_HOME=/root/.cache/huggingface + export HF_TOKEN= + export NUM_DEVICES= + pip install -e /opt/Cosmos + ``` + +3. In the `/opt` directory, run the inference script, choosing the index of the desired frame to visualize. Set the model size to one of 14B or 7B depending on the size of the post-trained model. +```bash + cd /opt + torchrun --nproc_per_node=$NUM_DEVICES \ + Cosmos/cosmos1/models/diffusion/nemo/inference/action_control_infer.py \ + /checkpoint/weights \ + --dataset-split val \ + --index \ + --output-dir outputs/ \ + --tensor-model-parallel-size 4 \ + --context-parallel-size 2 + --model-size + ``` + + +## Run inference Script on Camera-Control Models + +To run inference with a Camera-Control model, it is assumed users have first fine-tuned the Video2World base model for Camera-Control using the instructions provided in the [post-training README](../post_training/README.md). With a camera-control checkpoint generated during post-training, users can sample from this model and generate videos conditioned on camera intrinsics and extrinsics. This can be done as follows: + +```bash +export NVTE_FUSED_ATTN=0 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export NUM_DEVICES=1 +export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $((NUM_DEVICES - 1))) + +export NEMO_CHECKPOINT=nemo_experiments/cosmos_diffusion_7b_cameractrl_finetune/default/2025-03-17_15-57-48/checkpoints/epoch=23-step=399/weights + +export PROMPT="" +export CONDITIONED_VIDEO="" +export CAMERA_EXTRINSICS="" + +torchrun --nproc_per_node=1 cosmos1/models/diffusion/nemo/inference/camera_ctrl.py \ + --model Cosmos-1.0-Diffusion-7B-Video2World \ + --nemo_checkpoint "$NEMO_CHECKPOINT" \ + --cp_size 1 \ + --num_devices 1 \ + --video_save_path "camera_control_inference.mp4" \ + --guidance 7 \ + --seed 1 \ + --prompt "$PROMPT" \ + --conditioned_image_or_video_path "$CONDITIONED_IMAGE_OR_VIDEO" \ + --camera_extrinsics_path "$CAMERA_EXTRINSICS" \ + --num_video_frames 57 \ + --num_input_frames 9 + +``` + +An example for which we have verified camera control inference is the video and camera extrinsics corresponding to the `72e790b8c85027da48fa8c68142adfbc35c3ad8f2d48fcdf5bfb916927c987bd` hash within the `DL3DV-ALL-4K` subset of DL3DV. + + +### Multicamera Validation + +1. Install necessary dependencies + ```bash + pip install lru-dict pyquaternion git+https://github.com/NVIDIA/NeMo-Run.git@f07877f80dbfc91c4d37683864cf2552b96d62f1 + pip install -U moviepy==1.0.3 + ``` + +2. Modify Environment Variables + ```bash + export PYTHONPATH=$PYTHONPATH:/opt/NeMo/nemo/collections/physicalai/datasets/dataverse + export NVTE_FUSED_ATTN=0 + export CUDA_DEVICE_MAX_CONNECTIONS=1 + export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + export HF_TOKEN={'Huggingface Token'} + export HF_HOME={'Path to Huggingface Cache'} + export XDG_CACHE_HOME={'Dataset Cache'} + export PYTHONPATH=$PYTHONPATH:{'Path to Cosmos'} + export WANDB_API_KEY={'api key'} + export WANDB_PROJECT={'project name'} + ``` + +3. Run Multicamera Script + ```bash + torchrun --nproc_per_node=8 cosmos1/models/diffusion/nemo/inference/validate_multicamera.py + ``` diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/__init__.py new file mode 100644 index 00000000..b11bcc2d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/action_control_infer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/action_control_infer.py new file mode 100644 index 00000000..0c8fbe29 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/action_control_infer.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import torch +from cosmos1.models.autoregressive.nemo.post_training.action_control.prepare_dataset import Split +from cosmos1.models.diffusion.nemo.post_training.action_control.prepare_dataset import create_tokenizer + +from nemo import lightning as nl +from nemo.collections.diffusion.datamodule import ActionControlDiffusionDataset +from nemo.collections.diffusion.models.model import ( + DiT7BVideo2WorldActionConfig, + DiT14BVideo2WorldActionConfig, + DiTModel, +) +from nemo.collections.diffusion.sampler.cosmos.cosmos_extended_diffusion_pipeline import ExtendedDiffusionPipeline + + +DEFAULT_AUGMENT_SIGMA_LIST = 0.001 + + +def create_trainer(tensor_model_parallel_size: int, context_parallel_size: int): + """Initialize model parallel strategy. + Here, we only use CP2 because action control only has T=2 frames. + + Args: + tensor_model_parallel_size: number of the tensor model parallelism + context_parallel_size: context parallel size + Returns: + Trainer: The trainer object configured with fit strategy + """ + + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=tensor_model_parallel_size, + pipeline_model_parallel_size=1, + context_parallel_size=context_parallel_size, + pipeline_dtype=torch.bfloat16, + ) + + # Initialize ptl trainer + trainer = nl.Trainer( + devices=tensor_model_parallel_size * context_parallel_size, + max_steps=1, + accelerator="gpu", + strategy=strategy, + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + ) + return trainer + + +def setup_diffusion_pipeline( + nemo_ckpt: str, tensor_model_parallel_size: int, context_parallel_size: int, model_size: str +): + """ + Initialize the diffusion pipeline from the checkpoint. + + Args: + nemo_ckpt: Model checkpoint file + tensor_model_parallel_size: number of the tensor model parallelism + context_parallel_size: context parallel size + model_size: the size of the model. One of 7B or 14B. + + Returns: + ExtendedDiffusionPipeline: diffusion pipeline with the checkpointed model. + """ + + # Initialize the model with the action config + if model_size == "7B": + dit_config = DiT7BVideo2WorldActionConfig() + elif model_size == "14B": + dit_config = DiT14BVideo2WorldActionConfig() + else: + raise ValueError("Invalid model size. Choose '7B' or '14B'.") + + dit_model = DiTModel(dit_config) + + trainer = create_trainer(tensor_model_parallel_size, context_parallel_size) + + # Convert trainer to fabric for inference + fabric = trainer.to_fabric() + fabric.strategy.checkpoint_io.save_ckpt_format = "zarr" + fabric.strategy.checkpoint_io.validate_access_integrity = False + model = fabric.load_model(nemo_ckpt, dit_model).to(device="cuda", dtype=torch.bfloat16) + + # Initialize the diffusion pipeline + diffusion_pipeline = ExtendedDiffusionPipeline( + net=model.module, conditioner=model.conditioner, sampler_type="RES", seed=42 + ) + + return diffusion_pipeline + + +def run_diffusion_inference( + diffusion_pipeline: ExtendedDiffusionPipeline, + index: int = 0, + dataset_split: Split = Split.val, + output_dir: str = "outputs/", +): + """ + Generate an example action control prediction from the given diffusion pipeline. + Args: + diffusion_pipeline:diffusion pipeline with the checkpointed model. + index: The index of the item to use for the prediction. + dataset_split: The dataset split to use for the prediction. + output_dir: The directory to save the output images to. + + """ + + augment_sigma = DEFAULT_AUGMENT_SIGMA_LIST + + # Initialize data batch + acd = ActionControlDiffusionDataset(subfolder="diffusion", split=dataset_split.value) + data_batch = acd.collate_fn([acd[index]]) + data_batch = {k: v.cuda() if torch.is_tensor(v) else v for k, v in data_batch.items()} + data_batch["inference_fwd"] = True + + # Generate the next frame from the diffusion pipeline. + sample = diffusion_pipeline.generate_samples_from_batch( + data_batch, + guidance=7, + state_shape=data_batch["video"].squeeze(0).size(), + num_condition_t=data_batch["num_condition_t"], + condition_latent=data_batch["video"], + condition_video_augment_sigma_in_inference=augment_sigma, + ) + + # Post-processing and save image + if torch.distributed.get_rank() == 0: + sigma_data = 0.5 + + # Initialize the tokenizer + vae = create_tokenizer() + + predicted_sample = sample[:, :, 1:2] + first_sample = data_batch["video"][:, :, 0:1] + next_sample = data_batch["video"][:, :, 1:2] + + # Stack the input frame, ground truth next frame, and predicted next frame. + all_frames = torch.cat([first_sample, next_sample, predicted_sample]) + + # Decode the frames with the tokenizer into into RGB images together. + decoded_image = vae.decode(all_frames.cuda() / sigma_data).squeeze(2).detach() + grid = (1.0 + decoded_image).clamp(0, 2) / 2 + grid = (grid.permute(0, 2, 3, 1) * 255).to(torch.uint8).cpu().numpy().astype(np.uint8) + + # Create a single image with the three frames side by side. + _, ax = plt.subplots(nrows=1, ncols=3, figsize=(9, 3)) + ax[0].imshow(grid[0]) + ax[1].imshow(grid[1]) + ax[2].imshow(grid[2]) + text_labels = ["Input Frame", "Next Frame (Ground Truth)", "Next Frame (Predicted)"] + for i, text in enumerate(text_labels): + ax[i].set_title(text) + + output_path = Path(output_dir) / f"diffusion_control_{dataset_split.value}_{index}.png" + output_path.parent.mkdir(parents=True, exist_ok=True) + plt.savefig(output_path) + + +def main( + nemo_ckpt: str, + dataset_split: Split = Split.val, + index: int = 0, + output_dir: str = "outputs/", + tensor_model_parallel_size: int = 4, + context_parallel_size: int = 2, + model_size: str = "14B", +): + """ + Generate an example diffusion prediction from a post-trained checkpoint. + + Args: + checkpoint_directory: A path to a post-trained checkpoint directory. + dataset_split: The dataset split to use for the prediction. + index: The index of the item to use for the prediction. + output_dir: The directory to save the output images to. + tensor_model_parallel_size: number of the tensor model parallelism + context_parallel_size: context parallel size + model_size: the size of the model. One of 7B or 14B. + + """ + + diffusion_pipeline = setup_diffusion_pipeline( + nemo_ckpt, tensor_model_parallel_size, context_parallel_size, model_size + ) + run_diffusion_inference(diffusion_pipeline, index, dataset_split.val, output_dir) + + +if __name__ == "__main__": + import typer + + typer.run(main) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/control2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/control2world.py new file mode 100644 index 00000000..25970308 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/control2world.py @@ -0,0 +1,473 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os +from functools import partial + +import numpy as np +import torch +from huggingface_hub import snapshot_download +from megatron.core.dist_checkpointing.validation import StrictHandling +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + +from nemo import lightning as nl +from nemo.lightning.megatron_parallel import MegatronParallel + + +MegatronParallel.init_ddp = lambda self: None +from cosmos1.models.diffusion.conditioner import DataType +from cosmos1.models.diffusion.inference.inference_utils import read_video_or_image_into_frames_BCTHW +from cosmos1.models.diffusion.nemo.inference.inference_utils import get_ctrl_batch_nemo, process_prompt, save_video +from cosmos1.utils import log +from transformers import T5EncoderModel, T5TokenizerFast + +from nemo.collections.diffusion.mcore_parallel_utils import Utils +from nemo.collections.diffusion.sampler.conditioner import VideoExtendConditionerControl +from nemo.collections.diffusion.sampler.conditioner_configs import ( + FPSConfig, + ImageSizeConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, + VideoCondBoolConfig, +) +from nemo.collections.diffusion.sampler.cosmos.cosmos_control_diffusion_pipeline import CosmosControlDiffusionPipeline + + +EXAMPLE_PROMPT = ( + "The teal robot is cooking food in a kitchen. Steam rises from a simmering pot " + "as the robot chops vegetables on a worn wooden cutting board. Copper pans hang " + "from an overhead rack, catching glints of afternoon light, while a well-loved " + "cast iron skillet sits on the stovetop next to scattered measuring spoons and " + "a half-empty bottle of olive oil." +) + +DEFAULT_AUGMENT_SIGMA_LIST = 0.001 + + +""" +Example command to run the script: +export NUM_DEVICES=1 +export CONDITIONED_IMAGE_OR_VIDEO="cosmos1/models/diffusion/assets/v1p0/video2world_input1.mp4" +export PROMPT="The video showcases a vibrant green sports car with a sleek design, parked on a concrete surface. The car is oriented towards the right side of the frame, highlighting its rear and side profiles. The backdrop features a picturesque beach, with sandy dunes in the foreground and a serene sea in the background. The sky is clear, displaying a soft gradient of colors that hints at either sunrise or sunset. The scene is tranquil, with no people or other vehicles in sight, and the car remains stationary, inviting viewers to appreciate its beauty against the stunning natural landscape." +export CTRL_MODEL_DIR=/lustre/fsw/portfolios/coreai/users/sgovande/models/mcore_blur2world_7B_converted/ctrl +PYTHONPATH=$(pwd):~/lustre/nemo-vfm torchrun --nproc_per_node=$NUM_DEVICES cosmos1/models/diffusion/nemo/inference/control2world.py --cosmos_assets_dir ./checkpoints --conditioned_image_or_video_path $CONDITIONED_IMAGE_OR_VIDEO --num_devices $NUM_DEVICES --cp_size $NUM_DEVICES --prompt "$PROMPT" --control_weight 1 --ctrl_model_dir $CTRL_MODEL_DIR +""" + + +def parse_args(): + parser = argparse.ArgumentParser(description="Control2World foundation model inference") + parser.add_argument( + "--ctrl_model_dir", + type=str, + required=True, + ) + parser.add_argument( + "--base_model_dir", + type=str, + default="nvidia/Cosmos-1.0-Diffusion-7B-Video2World", + ) + parser.add_argument( + "--prompt", + type=str, + default=EXAMPLE_PROMPT, + help="Prompt which the sampled video condition on", + ) + parser.add_argument( + "--negative_prompt", + type=str, + default=( + "The video captures a series of frames showing ugly scenes, static with no motion, motion blur, " + "over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, " + "underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, " + "jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, " + "fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. " + "Overall, the video is of poor quality." + ), + help="Negative prompt which the sampled video condition on", + ) + parser.add_argument("--subject_name", type=str, default="", help="Name of fine-tuned subject") + parser.add_argument("--guidance", type=float, default=7, help="Classifier-free guidance scale") + parser.add_argument("--sampler", type=str, default="RES", help="Currently only supports RES sampler.") + parser.add_argument("--video_save_path", type=str, default="outputs/c2w.mp4", help="Path to save the video") + parser.add_argument("--fps", type=int, default=24, help="FPS of the sampled video") + parser.add_argument("--height", type=int, default=704, help="Height of image to sample") + parser.add_argument("--width", type=int, default=1280, help="Width of image to sample") + parser.add_argument("--seed", type=int, default=1, help="Random seed") + parser.add_argument("--num_devices", type=int, default=1, help="Number of devices for inference") + parser.add_argument("--cp_size", type=int, default=1, help="Number of cp ranks for multi-gpu inference.") + parser.add_argument("--num_steps", type=float, default=35, help="Number of diffusion sampling steps") + parser.add_argument("--num_video_frames", type=int, default=121, help="Number of video frames to sample") + parser.add_argument( + "--num_input_frames", type=int, default=1, help="Number of video frames for conditioning, [1-9]" + ) + parser.add_argument( + "--conditioned_image_or_video_path", type=str, default="", help="Path to conditioning video file." + ) + parser.add_argument("--tokenizer_dir", type=str, default="", help="Directory for video tokenizer") + parser.add_argument("--cosmos_assets_dir", type=str, default="", help="Directory containing cosmos assets") + parser.add_argument("--prompt_upsampler_dir", type=str, default="", help="Prompt upsampler weights directory") + parser.add_argument("--guardrail_dir", type=str, default="", help="Guardrails weights directory") + parser.add_argument("--nemo_checkpoint", type=str, default="", help="Video diffusion model nemo weights") + parser.add_argument("--t5_cache_dir", type=str, default=None, help="Path to T5 model") + parser.add_argument( + "--enable_prompt_upsampler", action="store_true", help="Whether to use prompt upsampling before generation" + ) + parser.add_argument("--control_weight", type=float, default=0.5, help="Control weight") + parser.add_argument( + "--blur_str", + type=str, + default="medium", + choices=["very_low", "low", "medium", "high", "very_high"], + help="blur strength applied to input", + ) + parser.add_argument("--no_preserve_color", action="store_true", help="Whether to not preserve color") + + args = parser.parse_args() + return args + + +def print_rank_0(string: str): + rank = torch.distributed.get_rank() + if rank == 0: + log.info(string) + + +@torch.no_grad() +def encode_for_batch(tokenizer: T5TokenizerFast, encoder: T5EncoderModel, prompts: list[str], max_length: int = 512): + """ + Encode a batch of text prompts to a batch of T5 embeddings. + Parameters: + tokenizer: T5 embedding tokenizer. + encoder: T5 embedding text encoder. + prompts: A batch of text prompts. + max_length: Sequence length of text embedding (defaults to 512). + """ + + batch_encoding = tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + # We expect all the processing is done on GPU. + input_ids = batch_encoding.input_ids.cuda() + attn_mask = batch_encoding.attention_mask.cuda() + + outputs = encoder(input_ids=input_ids, attention_mask=attn_mask) + encoded_text = outputs.last_hidden_state + + lengths = attn_mask.sum(dim=1).cpu() + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text + + +def init_video_tokenizer(args): + """ + Initializes video tokenizer based on specified video tokenizer config / path. + """ + from nemo.collections.diffusion.models.model import DiT7BConfig + + vae_path = os.path.join(args.cosmos_assets_dir, args.tokenizer_dir) + dit_config = DiT7BConfig(vae_path=vae_path) + vae = dit_config.configure_vae() + return vae + + +def check_prompt(args): + prompt = args.prompt + subject_string = None + if args.subject_name: + subject_string = f"A video of sks {args.subject_name}" + + prompt = process_prompt( + prompt=prompt, + checkpoint_dir=args.cosmos_assets_dir, + prompt_upsampler_dir=args.prompt_upsampler_dir, + guardrails_dir=args.guardrail_dir, + enable_prompt_upsampler=args.enable_prompt_upsampler, + ) + + if subject_string: + prompt = f"{subject_string}. {prompt}" + return prompt + + +def create_condition_latent_from_input_frames(tokenizer, input_frames, num_frames_condition=25): + B, C, T, H, W = input_frames.shape + num_frames_encode = tokenizer.pixel_chunk_duration + + assert input_frames.shape[2] >= num_frames_condition, ( + f"input_frames not enough for condition, require at least {num_frames_condition}, get {input_frames.shape[2]}, {input_frames.shape}" + ) + assert num_frames_encode >= num_frames_condition, ( + f"num_frames_encode should be larger than num_frames_condition, get {num_frames_encode}, {num_frames_condition}" + ) + + # Put the conditioal frames to the begining of the video, and pad the end with zero + condition_frames = input_frames[:, :, -num_frames_condition:] + padding_frames = condition_frames.new_zeros(B, C, num_frames_encode - num_frames_condition, H, W) + encode_input_frames = torch.cat([condition_frames, padding_frames], dim=2).to("cuda") + vae = tokenizer.to(encode_input_frames.device) + latent = vae.encode(encode_input_frames) + return latent, encode_input_frames + + +def prepare_data_batch(args, vae, t5_embeding_max_length=512): + tokenizer = T5TokenizerFast.from_pretrained("google-t5/t5-11b", cache_dir=args.t5_cache_dir) + text_encoder = T5EncoderModel.from_pretrained("google-t5/t5-11b", cache_dir=args.t5_cache_dir) + text_encoder.to("cuda") + text_encoder.eval() + + # Encode text to T5 embedding + out = encode_for_batch(tokenizer, text_encoder, [args.prompt])[0] + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + + # Padding T5 embedding to t5_embeding_max_length + L, C = encoded_text.shape + t5_embed = torch.zeros(1, t5_embeding_max_length, C, dtype=torch.bfloat16) + t5_embed[0, :L] = encoded_text + + if args.negative_prompt: + out = encode_for_batch(tokenizer, text_encoder, [args.negative_prompt])[0] + + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + # Padding T5 embedding to t5_embeding_max_length + L, C = encoded_text.shape + neg_t5_embed = torch.zeros(1, t5_embeding_max_length, C, dtype=torch.bfloat16) + neg_t5_embed[0, :L] = encoded_text + else: + neg_t5_embed = None + + # Prepare data sample + t, h, w = args.num_video_frames, args.height, args.width + state_shape = [ + vae.channel, + vae.get_latent_num_frames(t), + h // vae.spatial_compression_factor, + w // vae.spatial_compression_factor, + ] + + data_batch = { + "video": torch.zeros((1, 3, t, h, w), dtype=torch.uint8).cuda(), + "t5_text_embeddings": t5_embed, + "t5_text_mask": torch.ones(1, t5_embeding_max_length, dtype=torch.bfloat16).cuda(), + # other conditions + "image_size": torch.tensor( + [[args.height, args.width, args.height, args.width]] * 1, dtype=torch.bfloat16 + ).cuda(), + "fps": torch.tensor([args.fps] * 1, dtype=torch.bfloat16).cuda(), + "num_frames": torch.tensor([args.num_video_frames] * 1, dtype=torch.bfloat16).cuda(), + "padding_mask": torch.zeros((1, 1, args.height, args.width), dtype=torch.bfloat16).cuda(), + } + if args.negative_prompt: + data_batch["neg_t5_text_embeddings"] = neg_t5_embed + data_batch["neg_t5_text_mask"] = torch.ones(1, t5_embeding_max_length, dtype=torch.bfloat16) + + input_path_format = args.conditioned_image_or_video_path.split(".")[-1] + input_frames = read_video_or_image_into_frames_BCTHW( + args.conditioned_image_or_video_path, + input_path_format=input_path_format, + H=args.height, + W=args.width, + ) + + condition_latent, _ = create_condition_latent_from_input_frames(vae, input_frames, args.num_input_frames) + data_batch["condition_latent"] = condition_latent + data_batch["clean_image"] = condition_latent + + return data_batch, state_shape + + +def setup_diffusion_pipeline(args): + """ + Initialize DiT model, parallel strategy, and diffusion pipeline for inference. + """ + # Initialize DiT model + from nemo.collections.diffusion.models.model import DiT7BControl2WorldConfig, DiT7BVideo2WorldConfig, DiTModel + + dit_model = DiTModel(DiT7BControl2WorldConfig()) + dit_base_model = DiTModel(DiT7BVideo2WorldConfig()) + + # Initialize model parallel strategy. Here, we only use context parallel. + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=args.cp_size, + pipeline_dtype=torch.bfloat16, + ) + + # Initialize ptl trainer + trainer = nl.Trainer( + devices=args.num_devices, # you can change the numebr of devices to suit your setup + max_steps=1, + accelerator="gpu", + strategy=strategy, + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + ) + + # Convert trainer to fabric for inference + fabric = trainer.to_fabric() + fabric.strategy.checkpoint_io.save_ckpt_format = "zarr" + fabric.strategy.checkpoint_io.validate_access_integrity = False + fabric.strategy.checkpoint_io.load_checkpoint = partial( + fabric.strategy.checkpoint_io.load_checkpoint, strict=False + ) + StrictHandling.requires_global_app_metadata = staticmethod(lambda val: False) + + base_model = fabric.load_model(args.base_model_dir, dit_base_model).to(device="cuda", dtype=torch.bfloat16) + model = fabric.load_model(args.ctrl_model_dir, dit_model).to(device="cuda", dtype=torch.bfloat16) + + # Set up diffusion pipeline + conditioner = VideoExtendConditionerControl( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), + ) + + diffusion_pipeline = CosmosControlDiffusionPipeline( + net=model, base_model=base_model, conditioner=conditioner, sampler_type=args.sampler, seed=args.seed + ) + diffusion_pipeline.conditioner.data_type = DataType.VIDEO + + return diffusion_pipeline + + +def compute_num_latent_frames(tokenizer, num_input_frames: int, downsample_factor=8) -> int: + """This function computes the number of latent frames given the number of input frames. + Args: + tokenizer: video tokenizer + num_input_frames (int): number of input frames + downsample_factor (int): downsample factor for temporal reduce + Returns: + int: number of latent frames + """ + num_latent_frames = ( + num_input_frames // tokenizer.video_vae.pixel_chunk_duration * tokenizer.video_vae.latent_chunk_duration + ) + if num_input_frames % tokenizer.video_vae.latent_chunk_duration == 1: + num_latent_frames += 1 + elif num_input_frames % tokenizer.video_vae.latent_chunk_duration > 1: + assert (num_input_frames % tokenizer.video_vae.pixel_chunk_duration - 1) % downsample_factor == 0, ( + f"num_input_frames % tokenizer.video_vae.pixel_chunk_duration - 1 must be divisible by {downsample_factor}" + ) + num_latent_frames += 1 + (num_input_frames % tokenizer.video_vae.pixel_chunk_duration - 1) // downsample_factor + + return num_latent_frames + + +def run_diffusion_inference(args, data_batch, state_shape, vae, diffusion_pipeline): + # prepare data + data_batch = {k: v.cuda() if torch.is_tensor(v) else v for k, v in data_batch.items()} + data_batch["inference_fwd"] = True + augment_sigma = DEFAULT_AUGMENT_SIGMA_LIST + num_of_latent_condition = 0 # compute_num_latent_frames(vae, args.num_input_frames) + sample = diffusion_pipeline.generate_samples_from_batch( + data_batch, + guidance=args.guidance, + state_shape=state_shape, + num_steps=args.num_steps, + is_negative_prompt=True if "neg_t5_text_embeddings" in data_batch else False, + num_condition_t=num_of_latent_condition, + condition_latent=data_batch["condition_latent"], + condition_video_augment_sigma_in_inference=augment_sigma, + ) + + rank = torch.distributed.get_rank() + if rank == 0: + # Post-processing and save video + sigma_data = 0.5 + grid = (1.0 + vae.decode(sample / sigma_data)).clamp(0, 2) / 2 + grid = (grid[0].permute(1, 2, 3, 0) * 255).to(torch.uint8).cpu().numpy().astype(np.uint8) + save_video( + grid=grid, + fps=args.fps, + H=args.height, + W=args.width, + video_save_quality=5, + video_save_path=args.video_save_path, + checkpoint_dir=args.cosmos_assets_dir, + guardrails_dir=args.guardrail_dir, + ) + print_rank_0(f"saved video to {args.video_save_path}!") + + +def main(args): + if args.guardrail_dir == "": + args.guardrail_dir = snapshot_download("nvidia/Cosmos-1.0-Guardrail") + if args.tokenizer_dir == "": + args.tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + if args.prompt_upsampler_dir == "" and args.enable_prompt_upsampler: + args.prompt_upsampler_dir = snapshot_download("nvidia/Cosmos-1.0-Prompt-Upsampler-12B-Text2World") + if args.base_model_dir.startswith("nvidia/"): + args.base_model_dir = os.path.join(snapshot_download(args.base_model_dir, allow_patterns=["nemo/*"]), "nemo") + if args.ctrl_model_dir.startswith("nvidia/"): + args.ctrl_model_dir = os.path.join(snapshot_download(args.ctrl_model_dir, allow_patterns=["nemo/*"]), "nemo") + + # Initialize megatron model parallel environment + Utils.initialize_distributed(1, 1, context_parallel_size=args.cp_size) + model_parallel_cuda_manual_seed(args.seed) + + args.prompt = check_prompt(args) + + # Load video tokenizer + print_rank_0("initializing video tokenizer...") + vae = init_video_tokenizer(args) + + # Prepare data batch + print_rank_0("preparing data batch...") + data_batch, state_shape = prepare_data_batch(args, vae) + + print_rank_0("setting up diffusion pipeline...") + diffusion_pipeline = setup_diffusion_pipeline(args) + state_shape = [ + vae.channel, + vae.get_latent_num_frames(args.num_video_frames), + args.height // vae.spatial_compression_factor, + args.width // vae.spatial_compression_factor, + ] + diffusion_pipeline.net.module.module.module.sigma_data = diffusion_pipeline.sigma_data + data_batch = get_ctrl_batch_nemo( + diffusion_pipeline.net.module.module.module, + data_batch, + args.num_video_frames, + args.conditioned_image_or_video_path, + args.control_weight, + args.blur_str, + args.no_preserve_color, + state_shape, + vae.spatial_compression_factor, + hint_key="control_input_canny", + tokenizer=vae, + ) + print_rank_0("generating video...") + run_diffusion_inference(args, data_batch, state_shape, vae, diffusion_pipeline) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/general.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/general.py new file mode 100644 index 00000000..ce250840 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/general.py @@ -0,0 +1,366 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os +from functools import partial + +import numpy as np +import torch +from huggingface_hub import snapshot_download +from megatron.core.dist_checkpointing.validation import StrictHandling +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + +from nemo import lightning as nl +from nemo.lightning.megatron_parallel import MegatronParallel + + +MegatronParallel.init_ddp = lambda self: None +from cosmos1.models.diffusion.nemo.inference.inference_utils import process_prompt, save_video +from cosmos1.utils import log +from transformers import T5EncoderModel, T5TokenizerFast + +from nemo.collections.diffusion.mcore_parallel_utils import Utils +from nemo.collections.diffusion.sampler.conditioner import VideoConditioner +from nemo.collections.diffusion.sampler.conditioner_configs import ( + FPSConfig, + ImageSizeConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, +) +from nemo.collections.diffusion.sampler.cosmos.cosmos_diffusion_pipeline import CosmosDiffusionPipeline + + +EXAMPLE_PROMPT = ( + "The teal robot is cooking food in a kitchen. Steam rises from a simmering pot " + "as the robot chops vegetables on a worn wooden cutting board. Copper pans hang " + "from an overhead rack, catching glints of afternoon light, while a well-loved " + "cast iron skillet sits on the stovetop next to scattered measuring spoons and " + "a half-empty bottle of olive oil." +) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Video foundation model inference") + parser.add_argument( + "--model", + type=str, + default="Cosmos-1.0-Diffusion-7B-Text2World", + choices=["Cosmos-1.0-Diffusion-7B-Text2World", "Cosmos-1.0-Diffusion-14B-Text2World"], + ) + parser.add_argument( + "--prompt", + type=str, + default=EXAMPLE_PROMPT, + help="Prompt which the sampled video condition on", + ) + # We turn on negative prompt by default. set to "" to turn it off. + parser.add_argument( + "--negative_prompt", + type=str, + default=( + "The video captures a series of frames showing ugly scenes, static with no motion, motion blur, " + "over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, " + "underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, " + "jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, " + "fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. " + "Overall, the video is of poor quality." + ), + help="Negative prompt which the sampled video condition on", + ) + parser.add_argument("--subject_name", type=str, default="", help="Name of fine-tuned subject") + parser.add_argument("--guidance", type=float, default=7, help="Classifier-free guidance scale") + parser.add_argument("--sampler", type=str, default="RES", help="Currently only supports RES sampler.") + parser.add_argument("--video_save_path", type=str, default="outputs", help="Path to save the video") + parser.add_argument("--fps", type=int, default=24, help="FPS of the sampled video") + parser.add_argument("--height", type=int, default=704, help="Height of image to sample") + parser.add_argument("--width", type=int, default=1280, help="Width of image to sample") + parser.add_argument("--seed", type=int, default=1, help="Random seed") + parser.add_argument("--num_devices", type=int, default=1, help="Number of devices for inference") + parser.add_argument("--cp_size", type=int, default=1, help="Number of cp ranks for multi-gpu inference.") + parser.add_argument("--num_steps", type=float, default=35, help="Number of diffusion sampling steps") + parser.add_argument("--num_video_frames", type=int, default=121, help="Number of video frames to sample") + parser.add_argument("--tokenizer_dir", type=str, default="", help="Directory for video tokenizer") + parser.add_argument("--cosmos_assets_dir", type=str, default="", help="Directory containing cosmos assets") + parser.add_argument("--prompt_upsampler_dir", type=str, default="", help="Prompt upsampler weights directory") + parser.add_argument("--guardrail_dir", type=str, default="", help="Guardrails weights directory") + parser.add_argument("--nemo_checkpoint", type=str, default="", help="Video diffusion model nemo weights") + parser.add_argument("--t5_cache_dir", type=str, default=None, help="Path to T5 model") + parser.add_argument( + "--enable_prompt_upsampler", action="store_true", help="Whether to use prompt upsampling before generation" + ) + + args = parser.parse_args() + return args + + +def print_rank_0(string: str): + rank = torch.distributed.get_rank() + if rank == 0: + log.info(string) + + +@torch.no_grad() +def encode_for_batch(tokenizer: T5TokenizerFast, encoder: T5EncoderModel, prompts: list[str], max_length: int = 512): + """ + Encode a batch of text prompts to a batch of T5 embeddings. + Parameters: + tokenizer: T5 embedding tokenizer. + encoder: T5 embedding text encoder. + prompts: A batch of text prompts. + max_length: Sequence length of text embedding (defaults to 512). + """ + + batch_encoding = tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + # We expect all the processing is done on GPU. + input_ids = batch_encoding.input_ids.cuda() + attn_mask = batch_encoding.attention_mask.cuda() + + outputs = encoder(input_ids=input_ids, attention_mask=attn_mask) + encoded_text = outputs.last_hidden_state + + lengths = attn_mask.sum(dim=1).cpu() + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text + + +def init_video_tokenizer(args): + """ + Initializes video tokenizer based on specified video tokenizer config / path. + """ + from nemo.collections.diffusion.models.model import DiT7BConfig, DiT14BConfig + + vae_path = os.path.join(args.cosmos_assets_dir, args.tokenizer_dir) + dit_config = None + if "7b" in args.nemo_checkpoint.lower(): + dit_config = DiT7BConfig(vae_path=vae_path) + if "14b" in args.nemo_checkpoint.lower(): + dit_config = DiT14BConfig(vae_path=vae_path) + vae = dit_config.configure_vae() + return vae + + +def check_prompt(args): + prompt = args.prompt + subject_string = None + if args.subject_name: + subject_string = f"A video of sks {args.subject_name}" + + prompt = process_prompt( + prompt=prompt, + checkpoint_dir=args.cosmos_assets_dir, + prompt_upsampler_dir=args.prompt_upsampler_dir, + guardrails_dir=args.guardrail_dir, + enable_prompt_upsampler=args.enable_prompt_upsampler, + ) + + if subject_string: + prompt = f"{subject_string}. {prompt}" + return prompt + + +def prepare_data_batch(args, vae, t5_embeding_max_length=512): + tokenizer = T5TokenizerFast.from_pretrained("google-t5/t5-11b", cache_dir=args.t5_cache_dir) + text_encoder = T5EncoderModel.from_pretrained("google-t5/t5-11b", cache_dir=args.t5_cache_dir) + text_encoder.to("cuda") + text_encoder.eval() + + # Encode text to T5 embedding + out = encode_for_batch(tokenizer, text_encoder, [args.prompt])[0] + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + + # Padding T5 embedding to t5_embeding_max_length + L, C = encoded_text.shape + t5_embed = torch.zeros(1, t5_embeding_max_length, C, dtype=torch.bfloat16) + t5_embed[0, :L] = encoded_text + + if args.negative_prompt: + out = encode_for_batch(tokenizer, text_encoder, [args.negative_prompt])[0] + + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + # Padding T5 embedding to t5_embeding_max_length + L, C = encoded_text.shape + neg_t5_embed = torch.zeros(1, t5_embeding_max_length, C, dtype=torch.bfloat16) + neg_t5_embed[0, :L] = encoded_text + else: + neg_t5_embed = None + + # Prepare data sample + t, h, w = args.num_video_frames, args.height, args.width + state_shape = [ + vae.channel, + vae.get_latent_num_frames(t), + h // vae.spatial_compression_factor, + w // vae.spatial_compression_factor, + ] + + data_batch = { + "video": torch.zeros((1, 3, t, h, w), dtype=torch.uint8).cuda(), + "t5_text_embeddings": t5_embed, + "t5_text_mask": torch.ones(1, t5_embeding_max_length, dtype=torch.bfloat16).cuda(), + # other conditions + "image_size": torch.tensor( + [[args.height, args.width, args.height, args.width]] * 1, dtype=torch.bfloat16 + ).cuda(), + "fps": torch.tensor([args.fps] * 1, dtype=torch.bfloat16).cuda(), + "num_frames": torch.tensor([args.num_video_frames] * 1, dtype=torch.bfloat16).cuda(), + "padding_mask": torch.zeros((1, 1, args.height, args.width), dtype=torch.bfloat16).cuda(), + } + if args.negative_prompt: + data_batch["neg_t5_text_embeddings"] = neg_t5_embed + data_batch["neg_t5_text_mask"] = torch.ones(1, t5_embeding_max_length, dtype=torch.bfloat16) + + return data_batch, state_shape + + +def setup_diffusion_pipeline(args): + """ + Initialize DiT model, parallel strategy, and diffusion pipeline for inference. + """ + # Initialize DiT model + from nemo.collections.diffusion.models.model import DiT7BConfig, DiT14BConfig, DiTModel + + dit_config = None + if "7b" in args.nemo_checkpoint.lower(): + dit_config = DiT7BConfig() + if "14b" in args.nemo_checkpoint.lower(): + dit_config = DiT14BConfig() + + dit_model = DiTModel(dit_config) + + # Initialize model parallel strategy. Here, we only use context parallel. + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=args.cp_size, + pipeline_dtype=torch.bfloat16, + ) + + # Initialize ptl trainer + trainer = nl.Trainer( + devices=args.num_devices, # you can change the numebr of devices to suit your setup + max_steps=1, + accelerator="gpu", + strategy=strategy, + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + ) + + # Convert trainer to fabric for inference + fabric = trainer.to_fabric() + fabric.strategy.checkpoint_io.save_ckpt_format = "zarr" + fabric.strategy.checkpoint_io.validate_access_integrity = False + fabric.strategy.checkpoint_io.load_checkpoint = partial( + fabric.strategy.checkpoint_io.load_checkpoint, strict=False + ) + StrictHandling.requires_global_app_metadata = staticmethod(lambda val: False) + model = fabric.load_model(args.nemo_checkpoint, dit_model).to(device="cuda", dtype=torch.bfloat16) + + # Set up diffusion pipeline + conditioner = VideoConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + ) + diffusion_pipeline = CosmosDiffusionPipeline( + net=model.module, conditioner=conditioner, sampler_type=args.sampler, seed=args.seed + ) + + return diffusion_pipeline + + +def run_diffusion_inference(args, data_batch, state_shape, vae, diffusion_pipeline): + # prepare data + data_batch = {k: v.cuda() if torch.is_tensor(v) else v for k, v in data_batch.items()} + data_batch["inference_fwd"] = True + sample = diffusion_pipeline.generate_samples_from_batch( + data_batch, + guidance=args.guidance, + state_shape=state_shape, + num_steps=args.num_steps, + is_negative_prompt=True if "neg_t5_text_embeddings" in data_batch else False, + ) + + rank = torch.distributed.get_rank() + if rank == 0: + # Post-processing and save video + sigma_data = 0.5 + grid = (1.0 + vae.decode(sample / sigma_data)).clamp(0, 2) / 2 + grid = (grid[0].permute(1, 2, 3, 0) * 255).to(torch.uint8).cpu().numpy().astype(np.uint8) + save_video( + grid=grid, + fps=args.fps, + H=args.height, + W=args.width, + video_save_quality=5, + video_save_path=args.video_save_path, + checkpoint_dir=args.cosmos_assets_dir, + guardrails_dir=args.guardrail_dir, + ) + print_rank_0(f"saved video to {args.video_save_path}!") + + +def main(args): + if args.guardrail_dir == "": + args.guardrail_dir = snapshot_download("nvidia/Cosmos-1.0-Guardrail") + if args.tokenizer_dir == "": + args.tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + if args.prompt_upsampler_dir == "" and args.enable_prompt_upsampler: + args.prompt_upsampler_dir = snapshot_download("nvidia/Cosmos-1.0-Prompt-Upsampler-12B-Text2World") + if args.nemo_checkpoint == "": + args.nemo_checkpoint = snapshot_download(f"nvidia/{args.model}", allow_patterns=["nemo/*"]) + args.nemo_checkpoint = os.path.join(args.nemo_checkpoint, "nemo") + + # Initialize megatron model parallel environment + Utils.initialize_distributed(1, 1, context_parallel_size=args.cp_size) + model_parallel_cuda_manual_seed(args.seed) + + args.prompt = check_prompt(args) + + # Load video tokenizer + print_rank_0("initializing video tokenizer...") + vae = init_video_tokenizer(args) + + # Prepare data batch + print_rank_0("preparing data batch...") + data_batch, state_shape = prepare_data_batch(args, vae) + + # Setup model / diffusion pipeline + print_rank_0("setting up diffusion pipeline...") + diffusion_pipeline = setup_diffusion_pipeline(args) + + # Generate video from prompt + print_rank_0("generating video...") + run_diffusion_inference(args, data_batch, state_shape, vae, diffusion_pipeline) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/inference_utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/inference_utils.py new file mode 100644 index 00000000..dc2adf1d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/inference_utils.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import imageio +import numpy as np +import torch +from cosmos1.models.autoregressive.model import AutoRegressiveModel +from cosmos1.models.diffusion.config.ctrl.augmentors import BilateralOnlyBlurAugmentorConfig +from cosmos1.models.diffusion.datasets.augmentors.control_input import get_augmentor_for_eval +from cosmos1.models.diffusion.inference.inference_utils import read_video_or_image_into_frames_BCTHW +from cosmos1.models.diffusion.prompt_upsampler.text2world_prompt_upsampler_inference import ( + create_prompt_upsampler, + run_chat_completion, +) +from cosmos1.models.guardrail.common.presets import ( + create_text_guardrail_runner, + create_video_guardrail_runner, + run_text_guardrail, + run_video_guardrail, +) +from cosmos1.utils import log + + +def get_upsampled_prompt( + prompt_upsampler_model: AutoRegressiveModel, input_prompt: str, temperature: float = 0.01 +) -> str: + """ + Get upsampled prompt from the prompt upsampler model instance. + + Args: + prompt_upsampler_model: The prompt upsampler model instance. + input_prompt (str): Original prompt to upsample. + temperature (float): Temperature for generation (default: 0.01). + + Returns: + str: The upsampled prompt. + """ + dialogs = [ + [ + { + "role": "user", + "content": f"Upsample the short caption to a long caption: {input_prompt}", + } + ] + ] + + upsampled_prompt = run_chat_completion(prompt_upsampler_model, dialogs, temperature=temperature) + return upsampled_prompt + + +def print_rank_0(string: str): + rank = torch.distributed.get_rank() + if rank == 0: + log.info(string) + + +def process_prompt( + prompt: str, + checkpoint_dir: str, + prompt_upsampler_dir: str, + guardrails_dir: str, + image_path: str = None, + enable_prompt_upsampler: bool = True, +) -> str: + """ + Handle prompt upsampling if enabled, then run guardrails to ensure safety. + + Args: + prompt (str): The original text prompt. + checkpoint_dir (str): Base checkpoint directory. + prompt_upsampler_dir (str): Directory containing prompt upsampler weights. + guardrails_dir (str): Directory containing guardrails weights. + image_path (str, optional): Path to an image, if any (not implemented for upsampling). + enable_prompt_upsampler (bool): Whether to enable prompt upsampling. + + Returns: + str: The upsampled prompt or original prompt if upsampling is disabled or fails. + """ + + text_guardrail = create_text_guardrail_runner(os.path.join(checkpoint_dir, guardrails_dir)) + + # Check if the prompt is safe + is_safe = run_text_guardrail(str(prompt), text_guardrail) + if not is_safe: + raise ValueError("Guardrail blocked world generation.") + + if enable_prompt_upsampler: + if image_path: + raise NotImplementedError("Prompt upsampling is not supported for image generation") + else: + prompt_upsampler = create_prompt_upsampler( + checkpoint_dir=os.path.join(checkpoint_dir, prompt_upsampler_dir) + ) + upsampled_prompt = get_upsampled_prompt(prompt_upsampler, prompt) + print_rank_0(f"Original prompt: {prompt}\nUpsampled prompt: {upsampled_prompt}\n") + del prompt_upsampler + + # Re-check the upsampled prompt + is_safe = run_text_guardrail(str(upsampled_prompt), text_guardrail) + if not is_safe: + raise ValueError("Guardrail blocked world generation.") + + return upsampled_prompt + else: + return prompt + + +def save_video( + grid: np.ndarray, + fps: int, + H: int, + W: int, + video_save_quality: int, + video_save_path: str, + checkpoint_dir: str, + guardrails_dir: str, +): + """ + Save video frames to file, applying a safety check before writing. + + Args: + grid (np.ndarray): Video frames array [T, H, W, C]. + fps (int): Frames per second. + H (int): Frame height. + W (int): Frame width. + video_save_quality (int): Video encoding quality (0-10). + video_save_path (str): Output video file path. + checkpoint_dir (str): Directory containing model checkpoints. + guardrails_dir (str): Directory containing guardrails weights. + """ + video_classifier_guardrail = create_video_guardrail_runner(os.path.join(checkpoint_dir, guardrails_dir)) + + # Safety check on the entire video + grid = run_video_guardrail(grid, video_classifier_guardrail) + + kwargs = { + "fps": fps, + "quality": video_save_quality, + "macro_block_size": 1, + "ffmpeg_params": ["-s", f"{W}x{H}"], + "output_params": ["-f", "mp4"], + } + + imageio.mimsave(video_save_path, grid, "mp4", **kwargs) + + +def get_ctrl_batch_nemo( + model, + data_batch, + num_video_frames, + input_image_or_video_path, + control_weight, + blur_str, + no_preserve_color, + state_shape, + spatial_compression_factor, + hint_key="control_input_canny", + tokenizer=None, +): + """Prepare complete input batch for video generation including latent dimensions. + + Args: + model: Diffusion model instance + + Returns: + - data_batch (dict): Complete model input batch + """ + + H, W = ( + state_shape[-2] * spatial_compression_factor, + state_shape[-1] * spatial_compression_factor, + ) + + input_path_format = input_image_or_video_path.split(".")[-1] + input_frames = read_video_or_image_into_frames_BCTHW( + input_image_or_video_path, + input_path_format=input_path_format, + H=H, + W=W, + )[:, :, :num_video_frames] + T = input_frames.shape[2] + if T < num_video_frames: + pad_frames = input_frames[:, :, -1:].repeat(1, 1, num_video_frames - T, 1, 1) + input_frames = torch.cat([input_frames, pad_frames], dim=2) + + add_control_input = get_augmentor_for_eval( + input_key="video", + output_key=hint_key, + preset_strength=blur_str, + blur_config=BilateralOnlyBlurAugmentorConfig[blur_str], + ) + if no_preserve_color: + model.config.hint_mask = [True, False] + else: + model.config.hint_mask = [False, True] + + data_batch["hint_key"] = hint_key + data_batch["video"] = ((input_frames.cpu().float().numpy()[0] + 1) / 2 * 255).astype(np.uint8) + control_input = add_control_input(data_batch)[hint_key] + + data_batch["video"] = input_frames + data_batch[hint_key] = control_input[None].bfloat16().cuda() / 255 * 2 - 1 + if tokenizer is not None: + data_batch["latent_hint"] = model.encode_latent(data_batch, tokenizer) + else: + data_batch["latent_hint"] = model.encode_latent(data_batch) + data_batch["control_weight"] = control_weight + + return data_batch diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/validate_multicamera.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/validate_multicamera.py new file mode 100644 index 00000000..598f50a4 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/validate_multicamera.py @@ -0,0 +1,49 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +from cosmos1.models.diffusion.nemo.post_training.multicamera import cosmos_multicamera_diffusion_7b_text2world_finetune +from huggingface_hub import snapshot_download + +from nemo.collections import llm + + +@run.cli.factory(target=llm.validate) +def cosmos_multicamera_diffusion_7b_text2world_validate() -> run.Partial: + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + # Checkpoint load + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + return run.Partial( + llm.validate, + model=recipe.model, + data=recipe.data, + trainer=recipe.trainer, + log=recipe.log, + optim=recipe.optim, + tokenizer=None, + resume=recipe.resume, + model_transform=None, + ) + + +if __name__ == "__main__": + run.cli.main(llm.validate, default_factory=cosmos_multicamera_diffusion_7b_text2world_validate) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/video2world.py new file mode 100644 index 00000000..e6bb4fb3 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/inference/video2world.py @@ -0,0 +1,436 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os +from functools import partial + +import numpy as np +import torch +from huggingface_hub import snapshot_download +from megatron.core.dist_checkpointing.validation import StrictHandling +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + +from nemo import lightning as nl +from nemo.lightning.megatron_parallel import MegatronParallel + + +MegatronParallel.init_ddp = lambda self: None +from cosmos1.models.diffusion.conditioner import DataType +from cosmos1.models.diffusion.inference.inference_utils import read_video_or_image_into_frames_BCTHW +from cosmos1.models.diffusion.nemo.inference.inference_utils import process_prompt, save_video +from cosmos1.utils import log +from transformers import T5EncoderModel, T5TokenizerFast + +from nemo.collections.diffusion.mcore_parallel_utils import Utils +from nemo.collections.diffusion.sampler.conditioner import VideoExtendConditioner +from nemo.collections.diffusion.sampler.conditioner_configs import ( + FPSConfig, + ImageSizeConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, + VideoCondBoolConfig, +) +from nemo.collections.diffusion.sampler.cosmos.cosmos_extended_diffusion_pipeline import ExtendedDiffusionPipeline + + +EXAMPLE_PROMPT = ( + "The teal robot is cooking food in a kitchen. Steam rises from a simmering pot " + "as the robot chops vegetables on a worn wooden cutting board. Copper pans hang " + "from an overhead rack, catching glints of afternoon light, while a well-loved " + "cast iron skillet sits on the stovetop next to scattered measuring spoons and " + "a half-empty bottle of olive oil." +) + +DEFAULT_AUGMENT_SIGMA_LIST = 0.001 + + +def parse_args(): + parser = argparse.ArgumentParser(description="Video foundation model inference") + parser.add_argument( + "--model", + type=str, + default="Cosmos-1.0-Diffusion-7B-Video2World", + choices=["Cosmos-1.0-Diffusion-7B-Video2World", "Cosmos-1.0-Diffusion-14B-Video2World"], + ) + parser.add_argument( + "--prompt", + type=str, + default=EXAMPLE_PROMPT, + help="Prompt which the sampled video condition on", + ) + # We turn on negative prompt by default. set to "" to turn it off. + parser.add_argument( + "--negative_prompt", + type=str, + default=( + "The video captures a series of frames showing ugly scenes, static with no motion, motion blur, " + "over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, " + "underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, " + "jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, " + "fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. " + "Overall, the video is of poor quality." + ), + help="Negative prompt which the sampled video condition on", + ) + parser.add_argument("--subject_name", type=str, default="", help="Name of fine-tuned subject") + parser.add_argument("--guidance", type=float, default=7, help="Classifier-free guidance scale") + parser.add_argument("--sampler", type=str, default="RES", help="Currently only supports RES sampler.") + parser.add_argument("--video_save_path", type=str, default="outputs", help="Path to save the video") + parser.add_argument("--fps", type=int, default=24, help="FPS of the sampled video") + parser.add_argument("--height", type=int, default=704, help="Height of image to sample") + parser.add_argument("--width", type=int, default=1280, help="Width of image to sample") + parser.add_argument("--seed", type=int, default=1, help="Random seed") + parser.add_argument("--num_devices", type=int, default=1, help="Number of devices for inference") + parser.add_argument("--cp_size", type=int, default=1, help="Number of cp ranks for multi-gpu inference.") + parser.add_argument("--num_steps", type=float, default=35, help="Number of diffusion sampling steps") + parser.add_argument("--num_video_frames", type=int, default=121, help="Number of video frames to sample") + parser.add_argument( + "--num_input_frames", type=int, default=1, help="Number of video frames for conditioning, [1-9]" + ) + parser.add_argument( + "--conditioned_image_or_video_path", type=str, default="", help="Path to conditioning video file." + ) + parser.add_argument("--tokenizer_dir", type=str, default="", help="Directory for video tokenizer") + parser.add_argument("--cosmos_assets_dir", type=str, default="", help="Directory containing cosmos assets") + parser.add_argument("--prompt_upsampler_dir", type=str, default="", help="Prompt upsampler weights directory") + parser.add_argument("--guardrail_dir", type=str, default="", help="Guardrails weights directory") + parser.add_argument("--nemo_checkpoint", type=str, default="", help="Video diffusion model nemo weights") + parser.add_argument("--t5_cache_dir", type=str, default=None, help="Path to T5 model") + parser.add_argument( + "--enable_prompt_upsampler", action="store_true", help="Whether to use prompt upsampling before generation" + ) + + args = parser.parse_args() + return args + + +def print_rank_0(string: str): + rank = torch.distributed.get_rank() + if rank == 0: + log.info(string) + + +@torch.no_grad() +def encode_for_batch(tokenizer: T5TokenizerFast, encoder: T5EncoderModel, prompts: list[str], max_length: int = 512): + """ + Encode a batch of text prompts to a batch of T5 embeddings. + Parameters: + tokenizer: T5 embedding tokenizer. + encoder: T5 embedding text encoder. + prompts: A batch of text prompts. + max_length: Sequence length of text embedding (defaults to 512). + """ + + batch_encoding = tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + # We expect all the processing is done on GPU. + input_ids = batch_encoding.input_ids.cuda() + attn_mask = batch_encoding.attention_mask.cuda() + + outputs = encoder(input_ids=input_ids, attention_mask=attn_mask) + encoded_text = outputs.last_hidden_state + + lengths = attn_mask.sum(dim=1).cpu() + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text + + +def init_video_tokenizer(args): + """ + Initializes video tokenizer based on specified video tokenizer config / path. + """ + from nemo.collections.diffusion.models.model import DiT7BConfig + + vae_path = os.path.join(args.cosmos_assets_dir, args.tokenizer_dir) + dit_config = DiT7BConfig(vae_path=vae_path) + vae = dit_config.configure_vae() + return vae + + +def check_prompt(args): + prompt = args.prompt + subject_string = None + if args.subject_name: + subject_string = f"A video of sks {args.subject_name}" + + prompt = process_prompt( + prompt=prompt, + checkpoint_dir=args.cosmos_assets_dir, + prompt_upsampler_dir=args.prompt_upsampler_dir, + guardrails_dir=args.guardrail_dir, + enable_prompt_upsampler=args.enable_prompt_upsampler, + ) + + if subject_string: + prompt = f"{subject_string}. {prompt}" + return prompt + + +def create_condition_latent_from_input_frames(tokenizer, input_frames, num_frames_condition=25): + B, C, T, H, W = input_frames.shape + num_frames_encode = tokenizer.pixel_chunk_duration + + assert input_frames.shape[2] >= num_frames_condition, ( + f"input_frames not enough for condition, require at least {num_frames_condition}, get {input_frames.shape[2]}, {input_frames.shape}" + ) + assert num_frames_encode >= num_frames_condition, ( + f"num_frames_encode should be larger than num_frames_condition, get {num_frames_encode}, {num_frames_condition}" + ) + + # Put the conditioal frames to the begining of the video, and pad the end with zero + condition_frames = input_frames[:, :, -num_frames_condition:] + padding_frames = condition_frames.new_zeros(B, C, num_frames_encode - num_frames_condition, H, W) + encode_input_frames = torch.cat([condition_frames, padding_frames], dim=2).to("cuda") + vae = tokenizer.to(encode_input_frames.device) + latent = vae.encode(encode_input_frames) + return latent, encode_input_frames + + +def prepare_data_batch(args, vae, t5_embeding_max_length=512): + tokenizer = T5TokenizerFast.from_pretrained("google-t5/t5-11b", cache_dir=args.t5_cache_dir) + text_encoder = T5EncoderModel.from_pretrained("google-t5/t5-11b", cache_dir=args.t5_cache_dir) + text_encoder.to("cuda") + text_encoder.eval() + + # Encode text to T5 embedding + out = encode_for_batch(tokenizer, text_encoder, [args.prompt])[0] + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + + # Padding T5 embedding to t5_embeding_max_length + L, C = encoded_text.shape + t5_embed = torch.zeros(1, t5_embeding_max_length, C, dtype=torch.bfloat16) + t5_embed[0, :L] = encoded_text + + if args.negative_prompt: + out = encode_for_batch(tokenizer, text_encoder, [args.negative_prompt])[0] + + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + # Padding T5 embedding to t5_embeding_max_length + L, C = encoded_text.shape + neg_t5_embed = torch.zeros(1, t5_embeding_max_length, C, dtype=torch.bfloat16) + neg_t5_embed[0, :L] = encoded_text + else: + neg_t5_embed = None + + # Prepare data sample + t, h, w = args.num_video_frames, args.height, args.width + state_shape = [ + vae.channel, + vae.get_latent_num_frames(t), + h // vae.spatial_compression_factor, + w // vae.spatial_compression_factor, + ] + + data_batch = { + "video": torch.zeros((1, 3, t, h, w), dtype=torch.uint8).cuda(), + "t5_text_embeddings": t5_embed, + "t5_text_mask": torch.ones(1, t5_embeding_max_length, dtype=torch.bfloat16).cuda(), + # other conditions + "image_size": torch.tensor( + [[args.height, args.width, args.height, args.width]] * 1, dtype=torch.bfloat16 + ).cuda(), + "fps": torch.tensor([args.fps] * 1, dtype=torch.bfloat16).cuda(), + "num_frames": torch.tensor([args.num_video_frames] * 1, dtype=torch.bfloat16).cuda(), + "padding_mask": torch.zeros((1, 1, args.height, args.width), dtype=torch.bfloat16).cuda(), + } + if args.negative_prompt: + data_batch["neg_t5_text_embeddings"] = neg_t5_embed + data_batch["neg_t5_text_mask"] = torch.ones(1, t5_embeding_max_length, dtype=torch.bfloat16) + + input_path_format = args.conditioned_image_or_video_path.split(".")[-1] + input_frames = read_video_or_image_into_frames_BCTHW( + args.conditioned_image_or_video_path, + input_path_format=input_path_format, + H=args.height, + W=args.width, + ) + + condition_latent, _ = create_condition_latent_from_input_frames(vae, input_frames, args.num_input_frames) + data_batch["condition_latent"] = condition_latent + + return data_batch, state_shape + + +def setup_diffusion_pipeline(args): + """ + Initialize DiT model, parallel strategy, and diffusion pipeline for inference. + """ + # Initialize DiT model + from nemo.collections.diffusion.models.model import DiT7BVideo2WorldConfig, DiT14BVideo2WorldConfig, DiTModel + + dit_config = None + if "7B" in args.model: + dit_config = DiT7BVideo2WorldConfig() + elif "14B" in args.model: + dit_config = DiT14BVideo2WorldConfig() + dit_model = DiTModel(dit_config) + + # Initialize model parallel strategy. Here, we only use context parallel. + strategy = nl.MegatronStrategy( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + context_parallel_size=args.cp_size, + pipeline_dtype=torch.bfloat16, + ) + + # Initialize ptl trainer + trainer = nl.Trainer( + devices=args.num_devices, # you can change the numebr of devices to suit your setup + max_steps=1, + accelerator="gpu", + strategy=strategy, + plugins=nl.MegatronMixedPrecision(precision="bf16-mixed"), + ) + + # Convert trainer to fabric for inference + fabric = trainer.to_fabric() + fabric.strategy.checkpoint_io.save_ckpt_format = "zarr" + fabric.strategy.checkpoint_io.validate_access_integrity = False + fabric.strategy.checkpoint_io.load_checkpoint = partial( + fabric.strategy.checkpoint_io.load_checkpoint, strict=False + ) + StrictHandling.requires_global_app_metadata = staticmethod(lambda val: False) + + model = fabric.load_model(args.nemo_checkpoint, dit_model).to(device="cuda", dtype=torch.bfloat16) + + # Set up diffusion pipeline + conditioner = VideoExtendConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + video_cond_bool=VideoCondBoolConfig(), + ) + + diffusion_pipeline = ExtendedDiffusionPipeline( + net=model.module, conditioner=conditioner, sampler_type=args.sampler, seed=args.seed + ) + diffusion_pipeline.conditioner.data_type = DataType.VIDEO + + return diffusion_pipeline + + +def compute_num_latent_frames(tokenizer, num_input_frames: int, downsample_factor=8) -> int: + """This function computes the number of latent frames given the number of input frames. + Args: + tokenizer: video tokenizer + num_input_frames (int): number of input frames + downsample_factor (int): downsample factor for temporal reduce + Returns: + int: number of latent frames + """ + num_latent_frames = ( + num_input_frames // tokenizer.video_vae.pixel_chunk_duration * tokenizer.video_vae.latent_chunk_duration + ) + if num_input_frames % tokenizer.video_vae.latent_chunk_duration == 1: + num_latent_frames += 1 + elif num_input_frames % tokenizer.video_vae.latent_chunk_duration > 1: + assert (num_input_frames % tokenizer.video_vae.pixel_chunk_duration - 1) % downsample_factor == 0, ( + f"num_input_frames % tokenizer.video_vae.pixel_chunk_duration - 1 must be divisible by {downsample_factor}" + ) + num_latent_frames += 1 + (num_input_frames % tokenizer.video_vae.pixel_chunk_duration - 1) // downsample_factor + + return num_latent_frames + + +def run_diffusion_inference(args, data_batch, state_shape, vae, diffusion_pipeline): + # prepare data + data_batch = {k: v.cuda() if torch.is_tensor(v) else v for k, v in data_batch.items()} + data_batch["inference_fwd"] = True + augment_sigma = DEFAULT_AUGMENT_SIGMA_LIST + num_of_latent_condition = compute_num_latent_frames(vae, args.num_input_frames) + + sample = diffusion_pipeline.generate_samples_from_batch( + data_batch, + guidance=args.guidance, + state_shape=state_shape, + num_steps=args.num_steps, + is_negative_prompt=True if "neg_t5_text_embeddings" in data_batch else False, + num_condition_t=num_of_latent_condition, + condition_latent=data_batch["condition_latent"], + condition_video_augment_sigma_in_inference=augment_sigma, + ) + + rank = torch.distributed.get_rank() + if rank == 0: + # Post-processing and save video + sigma_data = 0.5 + grid = (1.0 + vae.decode(sample / sigma_data)).clamp(0, 2) / 2 + grid = (grid[0].permute(1, 2, 3, 0) * 255).to(torch.uint8).cpu().numpy().astype(np.uint8) + save_video( + grid=grid, + fps=args.fps, + H=args.height, + W=args.width, + video_save_quality=5, + video_save_path=args.video_save_path, + checkpoint_dir=args.cosmos_assets_dir, + guardrails_dir=args.guardrail_dir, + ) + print_rank_0(f"saved video to {args.video_save_path}!") + + +def main(args): + if args.guardrail_dir == "": + args.guardrail_dir = snapshot_download("nvidia/Cosmos-1.0-Guardrail") + if args.tokenizer_dir == "": + args.tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + if args.prompt_upsampler_dir == "" and args.enable_prompt_upsampler: + args.prompt_upsampler_dir = snapshot_download("nvidia/Cosmos-1.0-Prompt-Upsampler-12B-Text2World") + if args.nemo_checkpoint == "": + args.nemo_checkpoint = snapshot_download(f"nvidia/{args.model}", allow_patterns=["nemo/*"]) + args.nemo_checkpoint = os.path.join(args.nemo_checkpoint, "nemo") + + # Initialize megatron model parallel environment + Utils.initialize_distributed(1, 1, context_parallel_size=args.cp_size) + model_parallel_cuda_manual_seed(args.seed) + + args.prompt = check_prompt(args) + + # Load video tokenizer + print_rank_0("initializing video tokenizer...") + vae = init_video_tokenizer(args) + + # Prepare data batch + print_rank_0("preparing data batch...") + data_batch, state_shape = prepare_data_batch(args, vae) + + # Setup model / diffusion pipeline + print_rank_0("setting up diffusion pipeline...") + diffusion_pipeline = setup_diffusion_pipeline(args) + + # Generate video from prompt + print_rank_0("generating video...") + run_diffusion_inference(args, data_batch, state_shape, vae, diffusion_pipeline) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/README.md new file mode 100755 index 00000000..6098a289 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/README.md @@ -0,0 +1,510 @@ +# Cosmos Diffusion-based World Foundation Models: NeMo Framework User Guide + +Learn how to [post-train](#post-train) Cosmos Diffusion-based World Foundation Models (WFMs) using the [NVIDIA NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html) for your custom Physical AI tasks by following this guide. + +## Model Support Matrix + +The NeMo Framework supports the following Cosmos Diffusion models. Review the available models and their compute requirements for post-tuning and inference to determine the best model for your use case. + +| Model Name | Model Status | Compute Requirements for Post-Training | +|----------------------------------------------|------------------|------------------------------------------| +| Cosmos-1.0-Diffusion-7B-Text2World | **Supported** | 8 NVIDIA GPUs* | +| Cosmos-1.0-Diffusion-14B-Text2World | **Supported** | 8 NVIDIA GPUs* | +| Cosmos-1.0-Diffusion-7B-Video2World | **Supported** | 8 NVIDIA GPUs* | +| Cosmos-1.0-Diffusion-14B-Video2WorldB | **Supported** | 8 NVIDIA GPUs* | + + +**\*** `H100-80GB` or `A100-80GB` GPUs are recommended. + +## Post-Training Support Matrix + +Cosmos Diffusion-based WFMs can be post-trained for a variety of Physical AI tasks. Review the following table for a list of available Physical AI post-training tasks: + +| Post-training Task | Post-Training Support Status | +|-------------------------|--------------------| +| General post-training | **Supported** | +| Instruction control | **Supported** | +| Action control | **Supported** | +| Camera control | **Supported** | +| Multi-view generation | **Supported** | +| Multi-view generation with vehicle trajectory control | **Supported** | + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the minimum compute required to run the model(s), as listed in the model support matrix. + - **Containerization Platform**: We recommend using Docker with NVIDIA Container Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Clone the Cosmos Repository + +```bash +git clone git@github.com:NVIDIA/Cosmos.git +``` + +### 3. Start the Container + +The [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos Diffusion models. + +Run the following command to download and start the container: +```bash +docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidian/nemo:cosmos.1.0.2 bash +``` + +### 4. Download Checkpoints + +To help you get started, we've provided a [download script](../download_diffusion_nemo.py) to get the Cosmos Diffusion Text2World and Video2World checkpoints from Hugging Face. These checkpoints are in the NeMo distributed checkpoint format required to run post-training and inference with NeMo Framework. + +1. Set the following environment variables: + ```bash + # You must set HF_HOME before running this script. + export HF_TOKEN="" + export HF_HOME="" + ``` +2. Run the following command to download the models: + ```bash + cd /workspace/Cosmos + python cosmos1/models/diffusion/nemo/download_diffusion_nemo.py + ``` + +## Post-train + +Post-training a Cosmos Diffusion-based WFM enables you to train the model to generate videos that are more specific to your Physical AI use case. + +For example, if you want to generate action sequences for a specific robot, you can post-train the model to generate videos that are more aligned with typical actions/outcomes for that robot. + +There are 3 steps to post-training: preparing a dataset, preprocessing the data, and post-training the model. + +### 1. Prepare a Dataset + +The first step is to prepare a dataset. Post-training a Cosmos-1.0-Diffusion-Text2World/Cosmos-1.0-Diffusion-Video2World model enables you to generate videos of a specific subject in new environments using a collection of input videos of that same subject as reference material. + +You must provide a folder containing a collection of videos in **MP4 format**, preferably 720p. These videos should focus on the subject throughout the entire video so that each video chunk contains the subject. + +Run the following command to download the sample videos used for post-training: + +```bash +huggingface-cli download nvidia/Cosmos-NeMo-Assets --repo-type dataset --local-dir cosmos1/models/diffusion/assets/ --include "*.mp4*" +``` + +### 2. Preprocess Data for Single Subject Post-training + +The second step is to preprocess the input videos. This generates the post-training samples and the metadata required for the post-training process by: + +1. Selecting `N` chunks of 121 frames from each video, generating `N` post-training samples per video. +2. Encoding the 121 frames by first independently compressing the first frame and then applying an 8x temporal compression for the rest of the frames. +3. Generating `total_samples = # of videos x # of chunks` post-training samples. + +Before proceeding, ensure all videos are in **RGB format**. Complete the following steps to generate the post-training samples and metadata for the robot dataset. Remember to follow the given prompt format by including the subject's name in the prompt. For example, if the subject is "robot," the prompt should read `"A video of sks robot."`. + +1. Set the following environment variables: + ```bash + export HF_TOKEN="" + export HF_HOME="" + + # Path to Raw mp4 videos. + export RAW_DATA="cosmos1/models/diffusion/assets/nemo_diffusion_example_data" + + # Path to Processed Dataset. + export CACHED_DATA="./cached_data" && mkdir -p $CACHED_DATA + ``` + +2. Run the following command to preprocess the data: + + ```bash + python cosmos1/models/diffusion/nemo/post_training/prepare_dataset.py \ + --dataset_path $RAW_DATA \ + --output_path $CACHED_DATA \ + --prompt "A video of sks teal robot." \ + --num_chunks 500 + ``` + +Executing the [data preprocessing script](./prepare_dataset.py) generates the following files for each video (using `[i]` as the `index` of the video) at `$CACHED_DATA` path: + +- **`[i].info.json`**: Metadata for the video sample. +- **`[i].t5_text_embeddings.pth`**: T5-generated text embedding for the video clip. +- **`[i].t5_text_mask.pth`**: Mask for T5 text embedding, set to all ones by default to use the entire text embedding. +- **`[i].video_latent.pth`**: 3D spatiotemporal video tokens generated from the video tokenizer. +- **`[i].conditioning_latent.pth`**: 3D spatiotemporal video tokens generated from the video tokenizer on the first nine frames of the input video. These conditioning latents are only used during Video2World training. + +### 3. Preprocess Data for Robot Instruction (or other Custom Prompt) Post-training + +Robot instruction post-training uses instructions as input prompts. Instructions are imperative prompts and correspond to the physical actions performed by the robot in a video. The instruction dataset processing workflow generalizes to any custom input prompt per video. + +1. Create instruction dataset + +Create a dataset folder containing videos and per video instructions in the following format: + +``` +robot_dataset + videos + id1.mp4 + id2.mp4 + ... + instructions + id1.json + id2.json +``` + +- **`robot_dataset/videos/id1.mp4`**: video clip +- **`robot_dataset/instructions/id1.json`**: json file with key `language_instruction_0` mapping to a text instruction + +2. Run the following command to preprocess the data: + ```bash + python cosmos1/models/diffusion/nemo/post_training/prepare_instruction_dataset.py \ + --dataset_path robot_dataset \ + --output_path robot_dataset/processed \ + --num_chunks 500 + ``` +The output dataset is saved in `robot_dataset/processed/` in the same format described in the previous section. + +### 3. Post-train the Model + +The third step is to post-train the model. This step uses NeMo Framework's data and model parallelism capabilities to train the model on the post-training samples. This is accomplished by using utilizing Fully Sharded Data Parallel (FSDP) and Tensor Parallelism. + +- **FSDP**: Distributes model parameters, optimizer states, and activations across all GPUs +- **Tensor Parallelism**: Spreads the parameter tensor of individual layers across GPUs. + +> **NOTE**: +> For the 14B model, we also employ activation checkpointing to facilitate single-node training. + +#### Run the Post-training Script + + +Complete the following steps to post-train the Cosmos-1.0-Diffusion-7B-Text2World or Cosmos-1.0-Diffusion-7B-Video2World models on the robot dataset using 8 GPUs. + +##### Text2World + +1. Set the following environment variables: + ```bash + export HF_TOKEN="" + export HF_HOME="" + + # Optionally, you can monitor training progress with Weights and Biases (wandb). + export WANDB_API_KEY="" + export WANDB_PROJECT_NAME="cosmos-diffusion-nemo-post-training" + export WANDB_RUN_ID="cosmos_diffusion_7b_text2world_finetune" + ``` +2. Run the following command for Cosmos-Diffusion-Text2World-7B general post-training: + ```bash + NVTE_FUSED_ATTN=0 \ + CUDA_DEVICE_MAX_CONNECTIONS=1 \ + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + torchrun --nproc_per_node=8 cosmos1/models/diffusion/nemo/post_training/general.py \ + --yes \ + --factory cosmos_diffusion_7b_text2world_finetune \ + data.path=$CACHED_DATA \ + trainer.max_steps=1000 \ + optim.config.lr=1e-6 + ``` + +###### Configuration Options + +Before getting started, review the following parameters made available to the script. You can adjust these parameters to optimize performance based on your specific requirements. + +| Parameter | Description | Default | +|--------------------------------|---------------------------------------------------------------------------------|---------| +| `--factory` | recipe to use cosmos_diffusion_7b_text2world_finetune or cosmos_diffusion_14b_text2world_finetune for general post-training | cosmos_diffusion_7b_text2world_finetune | +| `data.path` | Path to processed post-training dataset (str). | None | +| `resume.restore_config.path` | Path to pre-trained Cosmos Diffusion NeMo distributed checkpoint (str). | None | +| `optim.config.lr` | Learning rate (float). | 1e-6 | + +##### Video2World + +1. Set the following environment variables: + ```bash + export HF_TOKEN="" + export HF_HOME="" + + # Optionally, you can monitor training progress with Weights and Biases (wandb). + export WANDB_API_KEY="" + export WANDB_PROJECT_NAME="cosmos-diffusion-nemo-post-training" + export WANDB_RUN_ID="cosmos_diffusion_7b_video2world_finetune" + ``` +2. Run the following command for Cosmos-Diffusion-Video2World-7B general post-training: + ```bash + NVTE_FUSED_ATTN=0 \ + CUDA_DEVICE_MAX_CONNECTIONS=1 \ + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + torchrun --nproc_per_node=8 cosmos1/models/diffusion/nemo/post_training/video2world.py \ + --yes \ + --factory cosmos_diffusion_7b_video2world_finetune \ + data.path=$CACHED_DATA \ + trainer.max_steps=1000 \ + optim.config.lr=1e-6 + ``` + +You can now run inference with your post-trained model using the instructions [here](../inference/README.md#run-the-inference-script-with-post-trained-model). + +###### Configuration Options + +Before getting started, review the following parameters made available to the script. You can adjust these parameters to optimize performance based on your specific requirements. + +| Parameter | Description | Default | +|--------------------------------|---------------------------------------------------------------------------------|---------| +| `--factory` | recipe to use cosmos_diffusion_7b_video2world_finetune or cosmos_diffusion_14b_video2world_finetune for video2world post-training | cosmos_diffusion_7b_video2world_finetune | +| `data.path` | Path to processed post-training dataset (str). | None | +| `resume.restore_config.path` | Path to pre-trained Cosmos Diffusion NeMo distributed checkpoint (str). | None | +| `optim.config.lr` | Learning rate (float). | 1e-6 | +| `trainer.max_steps` | Max number of post-training steps (int). | 1000 | +| `log.log_dir` | Path to folder to save post-training logs and checkpoints (str). | None | + + +## DiT Video2World Action Control Post-Training + +### Prerequisites + +#### Data Pre-Processing and Tokenization + +To preprocess and tokenize video data for diffusion-based action control, refer to the documentation in [action_control/README.md](./action_control/README.md). Diffusion datasets have a different continuous T5 video tokenizer than the auto-regressive model variant. The action control dataset should be downloaded to the default `HF_HOME=/root/.cache/huggingface` (or `HF_CACHE_DIR`), which we will mount to `HF_HOME=/root/.cache/huggingface` in our development containers. + +#### Diffusion-Based V2W Action Control Source Code + +The base container for DiT V2W action control can be downloaded with the following command: + +```bash +docker pull nvcr.io/nvidian/nemo:cosmos.1.0.2 +``` + +##### Cosmos + +Clone and checkout the `main` branch of Cosmos. + +```bash +git clone https://github.com/NVIDIA/Cosmos.git +cd Cosmos +COSMOS_CODE_DIR=$(pwd) +``` + +### Action Control Fine-Tuning + +When you have preprocessed datasets and model checkpoints cached in `HF_CACHE_DIR` or the default `HF_HOME=/root/.cache/huggingface`, as well as the above repositories cloned and checked out, we have all the resources necessary for action control fine-tuning. + +#### Setup + +Fine-tuning can be run with both Docker or Slurm. + +##### Docker Run + +To setup and interact with a Docker container for action control fine-tuning, run the following command: + +```bash +docker run --rm -it --name dit-action-ctrl-train-job --gpus all -v $COSMOS_CODE_DIR:/opt/Cosmos -v $HF_CACHE_DIR:$HF_HOME nvcr.io/nvidian/nemo:cosmos.1.0.2 +``` + +##### Slurm `srun` + +Requires an `enroot` container image, which you can build using the following command: + +```bash +enroot import -o cosmos-action-control-nemo-25.02rc3.sqsh docker://nvcr.io/nvidian/nemo:cosmos.1.0.2 +``` + +For instructions on how to install enroot on your system, refer to the Enroot documentation: https://github.com/NVIDIA/enroot. + +Then run the following command for interactive single-node training: + +```bash +srun --partition --account --nodes 1 --gpus-per-node --job-name --ntasks-per-node --time --container-mounts=$COSMOS_CODE_DIR:/opt/Cosmos,$HF_CACHE_DIR:$HF_HOME --container-image cosmos-action-control-nemo-25.02rc3.sqsh --exclusive --pty bash +``` + +#### Action Control Fine-Tuning + +Inside your container or compute node environment, run the following command to start fine-tuning the (7B) DiT model: + +```bash +export WANDB_API_KEY= # Existence of this ENV variable activates WandB logging when this is exported to the ENV. If not set, no logs will be uploaded to Weights & Biases. +export WANDB_NAME= +export WANDB_RUN_GROUP= +export WANDB_PROJECT= +export EXP_DIR= # Experiment directory that holds your logs and checkpoints for your run. Exposed here for easy access. +export HF_TOKEN= # Used to download models or the dataset. Not necessary if you already prepared your HuggingFace cache. + +# Change to our mounted Cosmos working directory. +cd /opt/Cosmos + +# Launch fine-tuning (in the Cosmos/ directory) with the `cosmos_diffusion_7b_video2world_action_ctrl_finetune` recipe! +torchrun --nproc_per_node 8 cosmos1/models/diffusion/nemo/post_training/video2world.py --yes --factory cosmos_diffusion_7b_video2world_action_ctrl_finetune data.global_batch_size=2 trainer.strategy.tensor_model_parallel_size=8 optim.config.lr=1e-5 +``` + +To review the complete set of parameters such as the learning rate, checkpointing strategy, model parameters, required video metadata and Tensor `dtype`, and more, the `cosmos_diffusion_{model_size}_video2world_action_ctrl_finetune` NeMo2 recipe and `DiTConfig` are located in these modules: + +- NeMo2 Recipe: `nemo-vfm/nemo/collections/diffusion/train.py::finetune_{7b,14b}_action_control` + - Training and checkpoint settings. +- `DiTConfig`: `nemo-vfm/nemo/collections/diffusion/models/model.py::DiT{7B,14B}Video2WorldActionConfig` + - Model architecture configuration, including the action control vector dimension, e.g. `config.action_emb_dim=7` (i.e. of shape `(B, 7)`) for the Bridge dataset. +- Data Module: `nemo-vfm/nemo/collections/diffusion/datamodule.py::ActionControlDiffusionDataset` + - Action control dataset and data loading. + +For more information on NeMo2 recipes and NeMo Lightning settings, you can refer to: https://github.com/NVIDIA/NeMo. + +#### Multi-Node Training + +For multi-node training, an example `sbatch .sbatch` script to fine-tune on 8 nodes with 8 GPU's per node is provided below: + +```bash +#!/bin/bash +#SBATCH -A coreai_dlalgo_llm +#SBATCH -p batch +#SBATCH -N 8 +#SBATCH --ntasks-per-node 8 +#SBATCH --gpus-per-node 8 +#SBATCH --gres=gpu:8 +#SBATCH --time 04:00:00 +#SBATCH --mail-type=FAIL +#SBATCH --exclusive # exclusive node access +#SBATCH --output=/path/to/slurm/job/logs/slurm_%x._%j.out +#SBATCH -J coreai_dlalgo_llm.cosmos-action-control-ar-post-training-20250225-8node + +# Mount our code into the base image. +export IMAGE= +COSMOS_CODE_DIR= +HF_CACHE_DIR= + +export WANDB_API_KEY= +export HF_TOKEN= +export MOUNT=$COSMOS_CODE_DIR:/opt/Cosmos,$HF_CACHE_DIR:/root/.cache/huggingface +export RUN_NAME="cosmos_action_control_diffusion_8node_globalbatch64_node_qatest4" +export MICRO_BATCH_SIZE=1 +export GLOBAL_BATCH_SIZE=$((SLURM_NNODES * $MICRO_BATCH_SIZE)) +export NUM_DEVICES=8 + +export nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIST ) ) +export nodes_array=($nodes) +export head_node=${nodes_array[0]} +export head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) + +read -r -d '' CMD < + export PYTHONPATH=/workspace/Cosmos:$PYTHONPATH + ``` + +3. Run the following command to download the models. We need the continuous video tokenizer model to tokenize the action control dataset. + + ```bash + python Cosmos/cosmos1/models/diffusion/nemo/download_diffusion_nemo.py + ``` + +4. Then run the `prepare_dataset` for the train, test, and val splits. The raw bridge dataset will +be downloaded automatically the first time you run it, and stored in +`HF_HOME/assets/cosmos/action-control/datasets/`. By default, the tokenized videos will be stored in +`HF_HOME/assets/cosmos/action-control/diffusion/bridge/`, which is used to train DiT V2W. If you +already have the bridge dataset downloaded to a different directory, then you can pass +`--dataset_dir` to point to the downloaded dataset. +> [!NOTE] +> The full bridge dataset is approximately 30Gb, and the full download can take approximately 2 +> hours depending on your connection speed. To use a smaller, 10Mb sanity-sized dataset bundled with +> the Cosmos repository, pass `--dataset-dir Cosmos/cosmos1/models/autoregressive/assets/bridge` to +> the commands below. + + ``` + python Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py \ + --dataset-split train --batch-size 50 --num-workers 16 + + python Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py \ + --dataset-split val --batch-size 50 --num-workers 16 + + python Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py \ + --dataset-split test --batch-size 50 --num-workers 16 + ``` diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py new file mode 100644 index 00000000..842872fb --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/action_control/prepare_dataset.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from typing import Literal + +import huggingface_hub +import torch +import typer +from cosmos1.models.autoregressive.nemo.post_training.action_control.prepare_dataset import ( + Split, + VideoDataset, + download_bridge_data, + get_annotations, + get_default_output_prefix, +) +from nemo.collections.diffusion.vae.video_vae import VideoJITTokenizer +from tqdm import tqdm + + +def create_tokenizer(tokenizer_tag: str = "nvidia/Cosmos-1.0-Tokenizer-CV8x8x8"): + """Creates a DiscreteVideoFSQJITTokenizer from a Hugging Face Hub tokenizer tag.""" + + tokenizer_path = Path(huggingface_hub.snapshot_download(tokenizer_tag)) + + name = "cosmos_tokenizer" + enc_fp = tokenizer_path / "encoder.jit" + dec_fp = tokenizer_path / "decoder.jit" + video_mean_std_fp = tokenizer_path / "mean_std.pt" + + image_vae = VideoJITTokenizer( + str(tokenizer_path), + str(enc_fp), + str(dec_fp), + name, + str(video_mean_std_fp), + pixel_chunk_duration=1, + spatial_compression_factor=8, + temporal_compression_factor=8, + ) + + return image_vae.cuda() + + +def get_tokenized_frames( + dataset_dir: Path, + dataset_split: Literal["train", "val", "test"], + tokenizer_tag: str, + batch_size: int, + num_workers: int, +): + """Tokenizes the video frames from the IRASim bridge dataset and saves them to the local filesystem.""" + + video_tokenizer = create_tokenizer(tokenizer_tag) + dataset = VideoDataset(dataset_dir, dataset_split) + dataloader = torch.utils.data.DataLoader( + dataset, + num_workers=num_workers, + batch_size=batch_size, + collate_fn=lambda x: torch.concat(x, dim=0), + ) + + def iter_tokenized_batches(): + for batch in tqdm(dataloader): + # Convert to bf16 and normalize from [0, 255] to [-1, 1] + batch = batch.to(device="cuda", dtype=torch.bfloat16, non_blocking=True) / 127.5 - 1.0 + latent_frames = video_tokenizer.encode(batch) + yield latent_frames.detach().to("cpu") + + with torch.no_grad(): + return torch.cat(list(iter_tokenized_batches()), dim=0) + + +def main( + tokenizer_tag: str = "nvidia/Cosmos-1.0-Tokenizer-CV8x8x8", + output_prefix: str | None = None, + dataset_split: Split = Split.train, + dataset_dir: str | None = None, + batch_size: int = 10, + num_workers: int = 10, +): + """Prepare the bridge dataset for diffusion post-training. + + This script will download the bridge dataset from lf-robot-opensource.bytetos.com, extract the annotations and + video frames, and tokenize the video frames using the specified tokenizer. The resulting files will be saved + to the local filesystem. + + Args: + tokenizer_tag: The tag of the tokenizer to be used in tokenizing the video frames. + output_prefix: The prefix to be used for the output files. If omitted, the output files will be saved to + the huggingface cache directory. + dataset_split: The split of the dataset to be processed. + dataset_dir: The path to the extracted contents of bridge_train_data.tar.gz, in the format provided by IRASim. + If omitted, the ~30Gb dataset will be downloaded from lf-robot-opensource.bytetos.com. + batch_size: The batch size (number of clips) to use during tokenization. + num_workers: The number of worker processes to use when processing the dataset. + """ + + if dataset_dir is None: + dataset_path = download_bridge_data() / "opensource_robotdata" / "bridge" + else: + dataset_path = Path(dataset_dir) + + if output_prefix is None: + output_path = get_default_output_prefix(dataset_split.value, subfolder="diffusion") + else: + output_path = Path(output_prefix) + + all_annotations = get_annotations(dataset_path, dataset_split.value, batch_size, num_workers) + latent_fames = get_tokenized_frames(dataset_path, dataset_split.value, tokenizer_tag, batch_size, num_workers) + + torch.save(all_annotations["state"], output_path / "state.pt") + torch.save(all_annotations["action"], output_path / "actions.pt") + torch.save(all_annotations["continuous_gripper_state"], output_path / "gripper.pt") + torch.save(latent_fames, output_path / "tokenized-frames.pt") + + +if __name__ == "__main__": + typer.run(main) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/camera_control/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/camera_control/README.md new file mode 100644 index 00000000..e7332aa5 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/camera_control/README.md @@ -0,0 +1,77 @@ +# Video2World DiT Camera Control Data Pre-Processing + +1. Run the following command to download and start the container: + + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + -v $HF_HOME:/root/.cache/huggingface \ + nvcr.io/nvidian/nemo:cosmos.1.0.2 bash + ``` + +2. Set the following environment variables: + + ``` + pip install --no-cache-dir imageio[ffmpeg] pyav iopath ffmpeg-python + export HF_HOME=/root/.cache/huggingface + export HF_TOKEN= + export PYTHONPATH=/workspace/Cosmos:$PYTHONPATH + ``` + +3. Run the following command to download the models. We need the continuous video tokenizer model to tokenize the camera control dataset. + + ```bash + python Cosmos/cosmos1/models/diffusion/nemo/download_diffusion_nemo.py + ``` + +4. To download the DL3DV datset, we recommend that users use the `scripts/download.py` script within the [DL3DV repo](https://github.com/DL3DV-10K/Dataset) which downloads the dataset from the HuggingFace Hub. + +5. Then run the `prepare_dataset.py` script to prepare the training samples for fine tuning the Cosmos-1.0-Video2World Difusion model for camera control. + + ```bash + # Path to flattened dataset + export RAW_DATA=/DL3DV/DL3DV-ALL-4K + + # Path to Processed Dataset. + export CACHED_DATA="./cached_camera_ctrl_data" && mkdir -p $CACHED_DATA + + # Path to intermediate processing cache + export PROCESSING_CACHE="./video_processing_cache" && mkdir -p $PROCESSING_CACHE + + python cosmos1/models/diffusion/nemo/post_training/camera_control/prepare_dataset.py \ + --dataset_path $RAW_DATA \ + --video_processing_cache_path $PROCESSING_CACHE \ + --output_path $CACHED_DATA \ + ``` + +By default, the above script will construct videos from the frames provided within `DL3DV/DL3DV-ALL-4K` zip files and write them to the `--video_processing_cache_path` (under the `video_files` dir). It will then extract one 57-frame chunk from each constructed video and the video latent using the [nvidia/Cosmos-1.0-Tokenizer-CV8X8X8](https://huggingface.co/nvidia/Cosmos-0.1-Tokenizer-CV8x8x8). We choose to construct the dataset in this manner to ensure that the videos align with the camera poses provided in the `transforms.json` files also included in the `DL3DV/DL3DV-ALL-4K` zip files. + +6. Upon completion of this script, users should observe files of the following format writtten to their `$CACHED_DATA` directory: + +```bash +0.video_latent.pth +0.t5_text_mask.pth +0.t5_text_embeddings.pth +0.plucker_embeddings.pth +0.padding_mask.pth +0.info.json +0.image_size.pth +0.conditioning_latent.pth +``` +where the first integer indicates the sample index, and the trailing text provides the description of the data associated with the sample. + + +## Configuration Options + +| Parameter | Description | Default | +|--------------------------------|---------------------------------------------------------------------------------|---------| +| `--dataset_path` | Path to DL3DV/DL3DV-ALL-4K dataset | None | +| `--video_processing_cache_path` | Path to where intermediate files will be stored | `./video_processing_cache` | +| `--path_to_caption_dict` | Path to a pickled Python dictionary with keys of `"/"` and values the caption for the video chunk. If a file is not provided, a dummy caption of "A video of a camera moving" will be used. | None | +| `--output_path` | Path to where the prepared output samples will be written | `./camera_ctrl_dataset_cached` | | +| `--num_chunks` | Number of random chunks to extract per zip file | 1 | +| `--height` | Height to resize constructed video file | 704 | +| `--width` | Width to resize constructed video file | 1280 | +| `--num_zip_files` | Number of DL3DV/DL3DV-ALL-4K zip files to use for creating the dataset. All used by default | -1 | +| `--seed` | Seed for randomly selecting chunks from video files | 1234 | +| `--tokenizer_dir` | Path to tokenizer dir | Downloaded from `nvidia/Cosmos-1.0-Tokenizer-CV8X8X8` to `$HF_HOME` | diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/camera_control/prepare_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/camera_control/prepare_dataset.py new file mode 100644 index 00000000..0399be14 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/camera_control/prepare_dataset.py @@ -0,0 +1,475 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import json +import os +import pickle +import random +import zipfile +from pathlib import Path +from typing import List + +import ffmpeg +import torch +import torchvision +import torchvision.transforms.functional as transforms_F +from cosmos1.models.diffusion.nemo.post_training.prepare_dataset import create_condition_latent_from_input_frames +from cosmos1.utils import log +from einops import rearrange +from huggingface_hub import snapshot_download +from nemo.collections.diffusion.data.camera_ctrl_utils import ( + estimate_pose_list_to_plucker_embedding, + normalize_camera_trajectory_to_unit_sphere, +) +from nemo.collections.diffusion.models.model import DiT7BCameraCtrlConfig +from tqdm import tqdm +from transformers import T5EncoderModel, T5TokenizerFast + + +def get_parser(): + parser = argparse.ArgumentParser(description="Process some configurations.") + parser.add_argument("--tokenizer_dir", type=str, default="", help="Path to the VAE model") + parser.add_argument( + "--dataset_path", + type=str, + default="", + help="Path to DL3DV dataset", + ) + parser.add_argument( + "--video_processing_cache_path", + type=str, + default="video_processing_cache", + help="Path to cache where frames will be extracted and used to form mp4 files", + ) + parser.add_argument( + "--path_to_caption_dict", + type=str, + default=None, + help="Path to a pickled python dictionary where the keys are the DL3DV sample hash followed by a chunk num " + "(e.g., '614d0e91913409a64d6d6c06a9d0c944d62d93b30b1d2d60493efd40ed4a5f9/2'), " + "and the values are the estimated captions for that chunk", + ) + parser.add_argument( + "--output_path", + type=str, + default="camera_ctrl_dataset_cached", + help="Path to the output directory", + ) + parser.add_argument( + "--num_chunks", + type=int, + default=1, + help="Number of random chunks to sample per video", + ) + parser.add_argument("--height", type=int, default=704, help="Height to resize video") + parser.add_argument("--width", type=int, default=1280, help="Width to resize video") + parser.add_argument( + "--seed", + type=int, + default=1234, + help="Seed for randomly selecting chunks", + ) + parser.add_argument( + "--num_zip_files", + type=int, + default=-1, + help="The number of zip files to use for creating the camera control dataset. " + "Default will be all zip files found recursively from the " + "'--dataset_path' directory ", + ) + return parser + + +def init_t5(): + """Initialize and return the T5 tokenizer and text encoder.""" + tokenizer = T5TokenizerFast.from_pretrained("google-t5/t5-11b") + text_encoder = T5EncoderModel.from_pretrained("google-t5/t5-11b") + text_encoder.to("cuda") + text_encoder.eval() + return tokenizer, text_encoder + + +def init_video_tokenizer(tokenizer_dir: str): + """Initialize and return the Cosmos Video tokenizer.""" + dit_config = DiT7BCameraCtrlConfig( + vae_path=tokenizer_dir, + ) + vae = dit_config.configure_vae() + return vae + + +@torch.no_grad() +def encode_for_batch(tokenizer, encoder, prompts: list[str], max_length=512): + """ + Encode a batch of text prompts to a batch of T5 embeddings. + Parameters: + tokenizer: T5 embedding tokenizer. + encoder: T5 embedding text encoder. + prompts: A batch of text prompts. + max_length: Sequence length of text embedding (defaults to 512). + """ + + batch_encoding = tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + # We expect all the processing is done on GPU. + input_ids = batch_encoding.input_ids.cuda() + attn_mask = batch_encoding.attention_mask.cuda() + + outputs = encoder(input_ids=input_ids, attention_mask=attn_mask) + encoded_text = outputs.last_hidden_state + + lengths = attn_mask.sum(dim=1).cpu() + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text + + +def prepare_raw_sample_from_zip( + input_zip_file: Path, + video_processing_cache_path: Path, +) -> List[Path]: + """ + Takes in a path to a zip file from the DL3DV-ALL-4K subset and performs the following: + 1. Creates a .mp4 file from the png frames and writes it to + video_processing_cache_path/video_files/.mp4, + where is the hash provided by DL3DV + 2. Extracts the transforms.json file to video_processing_cache_path/transform_files/_ + """ + # Make the output dirs + output_mp4_dir = video_processing_cache_path / "video_files" + output_mp4_dir.mkdir(exist_ok=True) + extracted_dir = video_processing_cache_path / "extracted_frames" + extracted_dir.mkdir(exist_ok=True) + transforms_dir = video_processing_cache_path / "transform_files" + transforms_dir.mkdir(exist_ok=True) + + # First prepare video (mp4) with ffmpeg (convert to 1280,720) + input_hash = input_zip_file.stem + output_mp4_file = output_mp4_dir / f"{input_hash}.mp4" + if not output_mp4_file.exists(): + # Get the hash of the zip file + # Extract the pngs from the zip file if they haven't been already + if not Path(extracted_dir / f"{input_hash}").exists(): + with zipfile.ZipFile(str(input_zip_file), "r") as zip_fp: + for ifile in zip_fp.infolist(): + if ifile.filename.lower().endswith(".png"): + zip_fp.extract(ifile, str(Path(extracted_dir))) + elif ifile.filename.lower().endswith(".json"): + zip_fp.extract(ifile, str(Path(transforms_dir))) + + # Prepare the transform file + transform_file = Path(transforms_dir / f"{input_hash}/transforms.json") + # Handle the case for no sub dir in the zip + if not transform_file.exists(): + transform_file = Path(transforms_dir / "transforms.json") + output_transform_file = transform_file.rename(transforms_dir / f"{input_hash}_transforms.json") + + # Remove the directory if it exists + if Path(transforms_dir / f"{input_hash}").exists(): + Path(transforms_dir / f"{input_hash}").rmdir() + + # Handle the case in which zip does not have a sub dir + if not Path(extracted_dir / f"{input_hash}").exists(): + Path(extracted_dir / f"{input_hash}/images").mkdir(exist_ok=True, parents=True) + Path(extracted_dir / "images").rename(extracted_dir / f"{input_hash}/images") + + # Build the input video file + input_pattern = str(extracted_dir / f"{input_hash}/images/*.png") + ( + ffmpeg.input(input_pattern, pattern_type="glob", framerate=7) + .filter("scale", 1280, 720) + .output(str(output_mp4_file), vcodec="libx264", preset="fast", crf=23) + .overwrite_output() + .run() + ) + else: + output_transform_file = Path(transforms_dir / f"{input_hash}_transforms.json") + if not output_transform_file.exists(): + output_transform_file.rename(transforms_dir / f"{input_hash}_transforms.json") + + return output_mp4_file, output_transform_file + + +def aspect_aware_resize_and_pad( + video_chunk: torch.Tensor, + H_target: int, + W_target: int, +) -> List[torch.Tensor]: + # [720, 1280] + _, _, H, W = video_chunk.shape + + # Aspect-ratio-aware resizing + # [720, 1280] -> [704, 1252] + scaling_ratio = H_target / H + H_resized = int(scaling_ratio * H + 0.5) + W_resized = int(scaling_ratio * W + 0.5) + video_chunk = transforms_F.resize( + video_chunk, + size=(H_resized, W_resized), + interpolation=transforms_F.InterpolationMode.BICUBIC, + antialias=True, + ) + + # Reflection padding + # [704, 1252] -> [704, 1280] + padding_left = int((W_target - W_resized) / 2) + padding_right = W_target - W_resized - padding_left + padding_top = int((H_target - H_resized) / 2) + padding_bottom = H_target - H_resized - padding_top + padding_vals = [padding_left, padding_top, padding_right, padding_bottom] + video_chunk = transforms_F.pad(video_chunk, padding_vals, padding_mode="reflect") + + # Construct the padding mask + padding_mask = torch.ones((1, H_target, W_target), dtype=torch.bfloat16) + padding_mask[:, padding_top : (padding_top + H), padding_left : (padding_left + W)] = 0 + image_size = torch.tensor( + [[H_target, W_target, H_resized, W_resized]] * 1, + dtype=torch.float16, + ) + + return video_chunk, padding_mask, image_size + + +def create_plucker_embeddings_from_pose_list( + camera_data: dict, + start_idx: int, + num_frames: int, + image_size: torch.tensor, + latent_compression_ratio_h: int = 8, + latent_compression_ratio_w: int = 8, +): + """ + Computes the plücker embeddings from an input list of poses + """ + # Prepare the instrinsics and extrinsics + intrinsic = [ + [camera_data["fl_x"], 0, camera_data["cx"]], + [0, camera_data["fl_y"], camera_data["cy"]], + [0, 0, 1], + ] + + pose_list = [] + for i, iframe in enumerate(camera_data["frames"]): + if i > start_idx and i <= start_idx + num_frames: + rotation, translation = [], [] + # Construct the 3 X 3 rotation matrix and the 3 X 1 translation vector + for j, row in enumerate(iframe["transform_matrix"]): + if j < 3: + rotation.append(row[:3]) + translation.append(row[-1]) + pose_list.append( + { + "intrinsics": intrinsic, + "rotation": rotation, + "translation": translation, + } + ) + + normalize_camera_trajectory_to_unit_sphere(pose_list) + plucker_coords, plucker_embeddings_h, plucker_embeddings_w = estimate_pose_list_to_plucker_embedding( + pose_list, + latent_compression_ratio_h, + latent_compression_ratio_w, + image_size, + ) + plucker_embeddings = rearrange( + plucker_coords, + "b (h w) c -> b c h w", + h=plucker_embeddings_h, + w=plucker_embeddings_w, + ) + + return plucker_embeddings.unsqueeze(0), plucker_embeddings_h, plucker_embeddings_w + + +def main(args): + # Set up output directory + os.makedirs(args.output_path, exist_ok=True) + os.makedirs(args.video_processing_cache_path, exist_ok=True) + + # Initialize T5 + tokenizer, text_encoder = init_t5() + + # Initialize the VAE + if args.tokenizer_dir == "": + args.tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + vae = init_video_tokenizer(args.tokenizer_dir) + + # Constants + t5_embeding_max_length = 512 + chunk_duration = vae.video_vae.pixel_chunk_duration # Frames per chunk + cnt = 0 # File index + + random.seed(args.seed) + + # Check if dataset_path is correct + files = list(Path(args.dataset_path).rglob("*.zip")) + if len(files) == 0: + raise ValueError(f"Dataset path {args.dataset_path} does not contain any .zip files.") + + if args.num_zip_files == -1: + num_zip_files = len(files) + else: + num_zip_files = args.num_zip_files + files = random.sample(files, num_zip_files) + + captions = None + if args.path_to_caption_dict is not None: + with open(args.path_to_caption_dict, "rb") as captions_fp: + captions = pickle.load(captions_fp) + + # Process each video in the dataset folder + with torch.no_grad(): + for zip_path in tqdm(files): + # Read video (T x H x W x C) + video_path, transform_path = prepare_raw_sample_from_zip( + zip_path, + Path(args.video_processing_cache_path), + ) + video, _, meta = torchvision.io.read_video(video_path) + T, H, W, C = video.shape + + with open(transform_path, "r") as transform_fp: + camera_data = json.load(transform_fp) + num_camera_frames = len(camera_data["frames"]) + + # Check to make sure we have a quality sample + if num_camera_frames != T: + log.info(f"Video camera_data is not aligned with video frames. Skipping {video_path}.") + continue + + # Skip videos shorter than one chunk + if T < chunk_duration: + log.info(f"Video {video_path} is shorter than {chunk_duration} frames. Skipped.") + continue + + # Sample random segments + for _ in range(args.num_chunks): + start_idx = random.randint(0, T - chunk_duration) + chunk = video[start_idx : start_idx + chunk_duration] # (chunk_duration, H, W, C) + + # Rearrange dimensions: (T, H, W, C) -> (T, C, H, W) + chunk = rearrange(chunk, "t h w c -> t c h w") + + # Aspect-aware resizing and padding to [704, 1280] for each frame + chunk, padding_mask, image_size = aspect_aware_resize_and_pad( + chunk, + args.height, + args.width, + ) + + # Expand dims: (T, C, H, W) -> (B=1, C, T, H, W) + chunk = rearrange(chunk, "(b t) c h w -> b c t h w", b=1) + + # Convert to bf16 and normalize from [0, 255] to [-1, 1] + chunk = chunk.to(device="cuda", dtype=torch.bfloat16, non_blocking=True) / 127.5 - 1.0 + + # Condition Latent (for Video2World training) + conditioning_chunk_len = 9 + conditioning_chunk = chunk[:, :, :conditioning_chunk_len, ...] + + # Encode video + latent = vae.encode(chunk).cpu() # shape: (1, latent_channels, T//factor, H//factor, W//factor) + + # Encode conditioning frames + conditioning_latent, _ = create_condition_latent_from_input_frames( + vae, conditioning_chunk, conditioning_chunk_len + ) + conditioning_latent = conditioning_latent.cpu() + + # Compute the Plücker embeddings from the camera data + ( + plucker_embeddings, + plucker_embeddings_h, + plucker_embeddings_w, + ) = create_plucker_embeddings_from_pose_list( + camera_data, + start_idx, + chunk_duration, + image_size[0], + ) + + padding_mask = padding_mask.unsqueeze(0) # 1,1,H_input,W_input + padding_mask = torch.nn.functional.interpolate( + padding_mask, size=(plucker_embeddings_h, plucker_embeddings_w), mode="nearest" + ) + plucker_embeddings = plucker_embeddings * (1 - padding_mask) + + # Get the corresponding caption for the 0th chunk of the video + if captions is not None: + sample_hash = zip_path.stem + key = f"{sample_hash}/0" + if key in captions: + prompt = captions[f"{sample_hash}/0"] + else: + log.info(f"Caption doesn't exist in provided captions file. Skipping {sample_hash}.") + continue + else: + prompt = "A video of a camera moving" + + # Encode text + out = encode_for_batch(tokenizer, text_encoder, [prompt])[0] + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + + # Pad T5 embedding to t5_embeding_max_length + L, C_ = encoded_text.shape + t5_embed = torch.zeros(1, t5_embeding_max_length, C_, dtype=torch.bfloat16) + t5_embed[0, :L] = encoded_text + + # Save data to folder + torch.save(latent[0], os.path.join(args.output_path, f"{cnt}.video_latent.pth")) + torch.save(conditioning_latent[0], os.path.join(args.output_path, f"{cnt}.conditioning_latent.pth")) + torch.save(plucker_embeddings[0], str(Path(args.output_path) / f"{cnt}.plucker_embeddings.pth")) + torch.save(image_size, str(Path(args.output_path) / f"{cnt}.image_size.pth")) + torch.save(padding_mask[0], str(Path(args.output_path) / f"{cnt}.padding_mask.pth")) + torch.save(t5_embed[0], os.path.join(args.output_path, f"{cnt}.t5_text_embeddings.pth")) + + # Create a T5 text mask of all ones + torch.save( + torch.ones(512, dtype=torch.bfloat16), os.path.join(args.output_path, f"{cnt}.t5_text_mask.pth") + ) + + # Save metadata + info = { + "height": H, + "width": W, + "fps": meta["video_fps"], + "num_frames": chunk_duration, + "video_path": os.path.basename(video_path), + "caption": prompt, + "start_frame": start_idx, + } + with open(os.path.join(args.output_path, f"{cnt}.info.json"), "w") as json_file: + json.dump(info, json_file) + + cnt += 1 + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/dataverse/av22_tar_index_full.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/dataverse/av22_tar_index_full.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/dataverse/av22_tar_index_full.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/general.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/general.py new file mode 100644 index 00000000..b24742dd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/general.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +from huggingface_hub import snapshot_download +from nemo.collections import llm +from nemo.collections.diffusion.models.model import DiT7BConfig, DiT14BConfig +from nemo.collections.diffusion.train import pretrain, videofolder_datamodule +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_7b_text2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(DiT7BConfig) + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.ckpt_load_strictness = "log_all" + + # FSDP + # recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + # recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = False + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.model.config.use_cpu_initialization = True + + # Activation Checkpointing + recipe.model.config.recompute_granularity = "full" + recipe.model.config.recompute_method = "uniform" + recipe.model.config.recompute_num_layers = 1 + + # Data setup + recipe.data = videofolder_datamodule() + recipe.data.path = "" # path to folder with processed dataset + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Text2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_diffusion_7b_text2world_finetune" + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_14b_text2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(DiT14BConfig) + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.ckpt_load_strictness = "log_all" + + # FSDP + # recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + # recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = False + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.model.config.use_cpu_initialization = True + + # Activation Checkpointing + recipe.model.config.recompute_granularity = "full" + recipe.model.config.recompute_method = "uniform" + recipe.model.config.recompute_num_layers = 1 + + # Data setup + recipe.data = videofolder_datamodule() + recipe.data.path = "" # path to folder with processed dataset + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-14B-Text2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_diffusion_14b_text2world_finetune" + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=cosmos_diffusion_7b_text2world_finetune) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/matplotlib/fontlist-v390.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/matplotlib/fontlist-v390.json new file mode 100644 index 00000000..7a2e7347 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/matplotlib/fontlist-v390.json @@ -0,0 +1,1074 @@ +{ + "_version": 390, + "_FontManager__default_weight": "normal", + "default_size": null, + "defaultFamily": { + "ttf": "DejaVu Sans", + "afm": "Helvetica" + }, + "afmlist": [ + { + "fname": "fonts/pdfcorefonts/Courier-Oblique.afm", + "name": "Courier", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Times-Bold.afm", + "name": "Times", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pcrb8a.afm", + "name": "Courier", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Times-BoldItalic.afm", + "name": "Times", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvro8a.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Helvetica.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pncbi8a.afm", + "name": "New Century Schoolbook", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pbkd8a.afm", + "name": "ITC Bookman", + "style": "normal", + "variant": "normal", + "weight": "demi", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pagd8a.afm", + "name": "ITC Avant Garde Gothic", + "style": "normal", + "variant": "normal", + "weight": "demi", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pagk8a.afm", + "name": "ITC Avant Garde Gothic", + "style": "normal", + "variant": "normal", + "weight": "book", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/ptmr8a.afm", + "name": "Times", + "style": "normal", + "variant": "normal", + "weight": "roman", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Times-Roman.afm", + "name": "Times", + "style": "normal", + "variant": "normal", + "weight": "roman", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pagdo8a.afm", + "name": "ITC Avant Garde Gothic", + "style": "italic", + "variant": "normal", + "weight": "demi", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pcrr8a.afm", + "name": "Courier", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Courier-Bold.afm", + "name": "Courier", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pbkli8a.afm", + "name": "ITC Bookman", + "style": "italic", + "variant": "normal", + "weight": "light", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pbkdi8a.afm", + "name": "ITC Bookman", + "style": "italic", + "variant": "normal", + "weight": "demi", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Helvetica-Oblique.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvb8a.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/ZapfDingbats.afm", + "name": "ZapfDingbats", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/ptmri8a.afm", + "name": "Times", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Courier.afm", + "name": "Courier", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvro8an.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "condensed", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pplri8a.afm", + "name": "Palatino", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Courier-BoldOblique.afm", + "name": "Courier", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pzdr.afm", + "name": "ITC Zapf Dingbats", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pzcmi8a.afm", + "name": "ITC Zapf Chancery", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/ptmbi8a.afm", + "name": "Times", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvr8a.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pplbi8a.afm", + "name": "Palatino", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvr8an.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "condensed", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/psyr.afm", + "name": "Symbol", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/cmmi10.afm", + "name": "Computer Modern", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Times-Italic.afm", + "name": "Times", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvlo8a.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "light", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pagko8a.afm", + "name": "ITC Avant Garde Gothic", + "style": "italic", + "variant": "normal", + "weight": "book", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/putbi8a.afm", + "name": "Utopia", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pncr8a.afm", + "name": "New Century Schoolbook", + "style": "normal", + "variant": "normal", + "weight": "roman", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvbo8an.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "condensed", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvbo8a.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/putr8a.afm", + "name": "Utopia", + "style": "normal", + "variant": "normal", + "weight": "regular", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pncri8a.afm", + "name": "New Century Schoolbook", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Helvetica-Bold.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pbkl8a.afm", + "name": "ITC Bookman", + "style": "normal", + "variant": "normal", + "weight": "light", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/putri8a.afm", + "name": "Utopia", + "style": "italic", + "variant": "normal", + "weight": "regular", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Helvetica-BoldOblique.afm", + "name": "Helvetica", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/cmsy10.afm", + "name": "Computer Modern", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/cmr10.afm", + "name": "Computer Modern", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pplb8a.afm", + "name": "Palatino", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pcrbo8a.afm", + "name": "Courier", + "style": "italic", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pncb8a.afm", + "name": "New Century Schoolbook", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/ptmb8a.afm", + "name": "Times", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/cmex10.afm", + "name": "Computer Modern", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pcrro8a.afm", + "name": "Courier", + "style": "italic", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/cmtt10.afm", + "name": "Computer Modern", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/pdfcorefonts/Symbol.afm", + "name": "Symbol", + "style": "normal", + "variant": "normal", + "weight": "medium", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/pplr8a.afm", + "name": "Palatino", + "style": "normal", + "variant": "normal", + "weight": "roman", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvl8a.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "light", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/putb8a.afm", + "name": "Utopia", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/afm/phvb8an.afm", + "name": "Helvetica", + "style": "normal", + "variant": "normal", + "weight": "bold", + "stretch": "condensed", + "size": "scalable", + "__class__": "FontEntry" + } + ], + "ttflist": [ + { + "fname": "fonts/ttf/STIXNonUniBol.ttf", + "name": "STIXNonUnicode", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizTwoSymReg.ttf", + "name": "STIXSizeTwoSym", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmb10.ttf", + "name": "cmb10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSansMono-Bold.ttf", + "name": "DejaVu Sans Mono", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmr10.ttf", + "name": "cmr10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXNonUniBolIta.ttf", + "name": "STIXNonUnicode", + "style": "italic", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSerif-BoldItalic.ttf", + "name": "DejaVu Serif", + "style": "italic", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXNonUniIta.ttf", + "name": "STIXNonUnicode", + "style": "italic", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmsy10.ttf", + "name": "cmsy10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizFiveSymReg.ttf", + "name": "STIXSizeFiveSym", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSans-Bold.ttf", + "name": "DejaVu Sans", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSans.ttf", + "name": "DejaVu Sans", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSans-BoldOblique.ttf", + "name": "DejaVu Sans", + "style": "oblique", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizFourSymReg.ttf", + "name": "STIXSizeFourSym", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSerifDisplay.ttf", + "name": "DejaVu Serif Display", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSerif-Bold.ttf", + "name": "DejaVu Serif", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmmi10.ttf", + "name": "cmmi10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSansMono.ttf", + "name": "DejaVu Sans Mono", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXGeneralItalic.ttf", + "name": "STIXGeneral", + "style": "italic", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXNonUni.ttf", + "name": "STIXNonUnicode", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmtt10.ttf", + "name": "cmtt10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmss10.ttf", + "name": "cmss10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/cmex10.ttf", + "name": "cmex10", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSansMono-BoldOblique.ttf", + "name": "DejaVu Sans Mono", + "style": "oblique", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSansDisplay.ttf", + "name": "DejaVu Sans Display", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXGeneral.ttf", + "name": "STIXGeneral", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizTwoSymBol.ttf", + "name": "STIXSizeTwoSym", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXGeneralBol.ttf", + "name": "STIXGeneral", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizOneSymReg.ttf", + "name": "STIXSizeOneSym", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSerif-Italic.ttf", + "name": "DejaVu Serif", + "style": "italic", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizOneSymBol.ttf", + "name": "STIXSizeOneSym", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSerif.ttf", + "name": "DejaVu Serif", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizThreeSymReg.ttf", + "name": "STIXSizeThreeSym", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSans-Oblique.ttf", + "name": "DejaVu Sans", + "style": "oblique", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizFourSymBol.ttf", + "name": "STIXSizeFourSym", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/DejaVuSansMono-Oblique.ttf", + "name": "DejaVu Sans Mono", + "style": "oblique", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXGeneralBolIta.ttf", + "name": "STIXGeneral", + "style": "italic", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "fonts/ttf/STIXSizThreeSymBol.ttf", + "name": "STIXSizeThreeSym", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSerif.ttf", + "name": "DejaVu Serif", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSerif-Bold.ttf", + "name": "DejaVu Serif", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", + "name": "DejaVu Sans", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", + "name": "DejaVu Sans Mono", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSansMono-BoldOblique.ttf", + "name": "DejaVu Sans Mono", + "style": "oblique", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSansMono-Oblique.ttf", + "name": "DejaVu Sans Mono", + "style": "oblique", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSansMono-Bold.ttf", + "name": "DejaVu Sans Mono", + "style": "normal", + "variant": "normal", + "weight": 700, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + }, + { + "fname": "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "name": "DejaVu Sans", + "style": "normal", + "variant": "normal", + "weight": 400, + "stretch": "normal", + "size": "scalable", + "__class__": "FontEntry" + } + ], + "__class__": "FontManager" +} \ No newline at end of file diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera.py new file mode 100644 index 00000000..cbfe9ffd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera.py @@ -0,0 +1,260 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os +from functools import partial + +import nemo_run as run +from huggingface_hub import snapshot_download +from lightning.pytorch.utilities.types import EVAL_DATALOADERS, TRAIN_DATALOADERS +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.train import pretrain +from nemo.collections.llm.gpt.data.mock import MockDataModule +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.alpamayo_dataloader import ( + InfiniteDataVerse, +) +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.config_dataverse import DATAVERSE_CONFIG +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.dataloader_utils import ( + dict_collation_fn, +) +from nemo.collections.physicalai.diffusion.post_training.multicamera.dit_multi_camera import ( + MultiCameraDiT7BConfig, + MultiCameraDiTModel, +) +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.strategies.utils import RestoreConfig +from platformdirs import user_cache_path +from torch.utils.data import DataLoader + + +class SimpleDataModule(MockDataModule): + def __init__(self, *args, dataset=None, **kwargs): + super().__init__(*args, **kwargs) + self.dataset = dataset + + def setup(self, stage: str = "") -> None: + self._train_ds = self.dataset() + self._validation_ds = self.dataset() + self._test_ds = self.dataset() + + def train_dataloader(self) -> TRAIN_DATALOADERS: + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds, num_workers=8) + + def val_dataloader(self) -> EVAL_DATALOADERS: + if not hasattr(self, "_validation_ds"): + self.setup() + return self._create_dataloader(self._validation_ds, num_workers=0) + + def _create_dataloader(self, dataset, num_workers=0, **kwargs) -> DataLoader: + return DataLoader( + dataset, + num_workers=num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers, + collate_fn=dict_collation_fn, + **kwargs, + ) + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model = run.Config( + MultiCameraDiTModel, + config=run.Config( + MultiCameraDiT7BConfig, + n_cameras=6, + camera_condition_dim=6, + add_repeat_frame_embedding=True, + # concat_traj_embedding=True, + # traj_condition_dim=12, + vae_path=snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8"), + pixel_chunk_duration=57, + recompute_granularity="full", + recompute_method="uniform", + recompute_num_layers=1, + ), + ) + recipe.trainer.strategy.ckpt_load_strictness = False + recipe.trainer.val_check_interval = 100 + recipe.trainer.limit_val_batches = 1 + + # Trainer setup + recipe.trainer.max_steps = 30000 + + # Optim setup + recipe.optim.config.lr = 1e-4 + recipe.optim.config.weight_decay = 0.3 + recipe.optim.config.adam_eps = 1e-8 + recipe.optim.config.adam_beta1 = 0.9 + recipe.optim.config.adam_beta2 = 0.999 + recipe.optim.lr_scheduler = run.Config( + nl.lr_scheduler.WarmupHoldPolicyScheduler, warmup_steps=1000, min_lr=1.0e-6, hold_steps=1e9 + ) + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.context_parallel_size = 1 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.save_ckpt_format = "torch_dist" + + # FSDP + # recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + # recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = False + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.model.config.use_cpu_initialization = True + + recipe.trainer.callbacks = [ + run.Config( + ModelCheckpoint, + monitor="reduced_train_loss", + dirpath="nemo_experiments/cosmos_multicamera_diffusion_7b_text2world_finetune/default/experiment_dir", + filename="{epoch}-{step}", + every_n_train_steps=100, + save_top_k=5, + always_save_context=True, + save_weights_only=False, + ), + run.Config(PreemptionCallback), + ] + + # Data setup + recipe.data = run.Config( + SimpleDataModule, + micro_batch_size=1, + global_batch_size=1, + dataset=partial(InfiniteDataVerse, **DATAVERSE_CONFIG["alpamayo_v2_traj_qwen_24fps_6_cameras_frame_repeat"]), + ) + + recipe.resume = run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + resume_from_directory="nemo_experiments/cosmos_multicamera_diffusion_7b_text2world_finetune/default/experiment_dir", + ) + + # Checkpoint load + recipe.resume.restore_config = run.Config( + RestoreConfig, + path=os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Text2World", allow_patterns=["nemo/*"]), "nemo" + ), # path to diffusion model checkpoint + load_model_state=True, + load_optim_state=False, + load_artifacts=False, + ) + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_overfit() -> run.Partial: + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + # Path to overfit data + dataset_path = str(user_cache_path("AV-V2.2")) + # dataset_path = "./overfit_data" + tar_dirs_training = [f"{dataset_path}/trainv2-2-chunk-00"] + + # Custom Dataset Config + ds_config = DATAVERSE_CONFIG["alpamayo_v2_traj_qwen_24fps_6_cameras_frame_repeat"] + ds_config["dataset_cfg"]["params"]["tar_dirs"] = tar_dirs_training + ds_config["dataset_cfg"]["params"]["uuid_dirs"] = None + ds_config["dataset_cfg"]["params"]["t5_dirs"] = f"{dataset_path}/alpamayo_caption_t5/qwen_t5_tars/" + + # Data setup + recipe.data = run.Config( + SimpleDataModule, + micro_batch_size=1, + global_batch_size=1, + dataset=partial(InfiniteDataVerse, **ds_config), + ) + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune_w_traj() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + recipe.model.config.concat_traj_embedding = True + recipe.model.config.traj_condition_dim = 12 + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune_w_traj_debug() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune_w_traj() + + recipe.model.config.concat_traj_embedding = True + recipe.model.config.traj_condition_dim = 12 + recipe.model.config.num_layers = 1 + recipe.resume.restore_config = None + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_image2world_finetune() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + recipe.model = run.Config( + MultiCameraDiTModel, + config=run.Config( + MultiCameraDiT7BConfig, + n_cameras=6, + camera_condition_dim=6, + add_repeat_frame_embedding=True, + # concat_traj_embedding=True, + # traj_condition_dim=12, + vae_path=snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8"), + pixel_chunk_duration=57, + recompute_granularity="full", + recompute_method="uniform", + recompute_num_layers=1, + ), + ) + + # Checkpoint load + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_image2world_finetune_w_traj() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_image2world_finetune() + + recipe.model.config.concat_traj_embedding = True + recipe.model.config.traj_condition_dim = 32 + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=cosmos_multicamera_diffusion_7b_text2world_finetune) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/mapping.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/mapping.json new file mode 100644 index 00000000..3b433019 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/mapping.json @@ -0,0 +1,8 @@ +{ + "1a1c7950-01f0-4e37-9949-dad68e3d1072.camera_front_wide_120fov": "cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar", + "1a1c7950-01f0-4e37-9949-dad68e3d1072.camera_cross_left_120fov": "cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar", + "1a1c7950-01f0-4e37-9949-dad68e3d1072.camera_cross_right_120fov": "cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar", + "1a1c7950-01f0-4e37-9949-dad68e3d1072.camera_rear_tele_30fov": "cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar", + "1a1c7950-01f0-4e37-9949-dad68e3d1072.camera_rear_left_70fov": "cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar", + "1a1c7950-01f0-4e37-9949-dad68e3d1072.camera_rear_right_70fov": "cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar" +} \ No newline at end of file diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.json b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.json new file mode 100644 index 00000000..8f56a7df --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.json @@ -0,0 +1 @@ +{"1a1c7950-01f0-4e37-9949-dad68e3d1072":{"0":["trainv2-2-chunk-00",["The video shows a car driving on a straight road at night. The car's trajectory is consistent, moving forward without any turns. The scene is illuminated by streetlights, and the road appears to be clear of other vehicles or pedestrians. The environment is a well-lit urban area, with no visible weather conditions affecting the visibility. The time of day is nighttime, as indicated by the darkness and artificial lighting."],[2],[130]],"1":["trainv2-2-chunk-00",["The video shows a car driving on a road at night. The car's trajectory indicates a steady forward movement. The scene is illuminated by streetlights, and the road appears to be clear of other vehicles and pedestrians. The weather conditions seem to be clear, with no visible rain or fog. The time of day is nighttime, as indicated by the darkness and artificial lighting."],[2],[130]],"2":["trainv2-2-chunk-00",["The video shows a car driving on a road at night. The car's trajectory indicates it is moving forward. The scene includes streetlights illuminating the road, and there are no other visible vehicles or pedestrians. The environment appears to be a quiet, possibly suburban area with minimal traffic. The weather conditions are not clearly visible, but the lighting suggests it is nighttime."],[2],[130]],"3":["trainv2-2-chunk-00",["The video depicts a car driving on a road at night. The car is moving forward, and the trajectory shows a steady path. The scene is illuminated by streetlights, casting a yellowish glow. Several cars are visible in the distance, and the road appears to be relatively empty. The weather seems clear, with no visible signs of rain or fog. The time of day is nighttime, as indicated by the artificial lighting and the darkness of the surroundings."],[2],[130]]}} \ No newline at end of file diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar new file mode 100644 index 00000000..1dd6cd76 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera/AV-V2.2/alpamayo_caption_t5/qwen_t5_tars/trainv2-2-chunk-00/part_000000/001365.tar differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera_mock.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera_mock.py new file mode 100644 index 00000000..84611f9f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/multicamera_mock.py @@ -0,0 +1,131 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +import torch +from huggingface_hub import snapshot_download +from nemo.collections import llm +from nemo.collections.diffusion.datamodule import DiTDataModule +from nemo.collections.diffusion.train import pretrain +from nemo.collections.physicalai.diffusion.post_training.multicamera.dit_multi_camera import MultiCameraDiT7BConfig +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + + +class MultiCameraDiTVideoLatentMockDataset(torch.utils.data.Dataset): + def __init__(self, num_samples, seq_len=21760): + self.length = num_samples if num_samples > 0 else 1 << 32 + self.seq_len = seq_len + + def __len__(self): + return self.length + + def __getitem__(self, idx): + t = 16 + h = 34 + w = 40 + seq_len = t * h * w + video_latent = torch.randn(16, t, h, w).to(dtype=torch.uint8) + loss_mask = torch.ones(seq_len, dtype=torch.bfloat16) + noise_latent = torch.rand_like(video_latent, dtype=torch.bfloat16) + timesteps = torch.randn(1, dtype=torch.bfloat16) + text_embedding = torch.randn(512, 1024, dtype=torch.bfloat16) + + return { + "video": video_latent, + "noise_latent": noise_latent, + "timesteps": timesteps, + "t5_text_embeddings": text_embedding, + "t5_text_mask": torch.ones(512, dtype=torch.bfloat16), + "image_size": torch.tensor([[34, 40, 34, 40]] * 1, dtype=torch.bfloat16), + "fps": torch.tensor([30] * 1, dtype=torch.bfloat16), + "num_frames": torch.tensor([16] * 1, dtype=torch.bfloat16), + "padding_mask": torch.zeros((1, 1, 34, 40), dtype=torch.bfloat16), + "loss_mask": loss_mask, + } + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(MultiCameraDiT7BConfig) + recipe.trainer.strategy.ckpt_load_strictness = False + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + + # FSDP + recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = True + recipe.trainer.strategy.ddp.overlap_grad_reduce = True + recipe.model.config.use_cpu_initialization = True + + # Data setup + recipe.data = DiTDataModule( + dataset=MultiCameraDiTVideoLatentMockDataset, path=1000, micro_batch_size=1, global_batch_size=1 + ) + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Text2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_multicamera_diffusion_7b_text2world_finetune" + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=cosmos_multicamera_diffusion_7b_text2world_finetune) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/prepare_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/prepare_dataset.py new file mode 100644 index 00000000..6e83f0a8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/prepare_dataset.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import glob +import json +import os +import random + +import torch +import torchvision +from cosmos1.utils import log +from einops import rearrange +from huggingface_hub import snapshot_download +from nemo.collections.diffusion.models.model import DiT7BConfig +from tqdm import tqdm +from transformers import T5EncoderModel, T5TokenizerFast + + +def get_parser(): + parser = argparse.ArgumentParser(description="Process some configurations.") + parser.add_argument("--tokenizer_dir", type=str, default="", help="Path to the VAE model") + parser.add_argument( + "--dataset_path", type=str, default="video_dataset", help="Path to the dataset (a folder of videos)" + ) + parser.add_argument("--output_path", type=str, default="video_dataset_cached", help="Path to the output directory") + parser.add_argument("--prompt", type=str, default="a video of sks.", help="Prompt for the video") + parser.add_argument("--num_chunks", type=int, default=5, help="Number of random chunks to sample per video") + parser.add_argument("--height", type=int, default=704, help="Height to resize video") + parser.add_argument("--width", type=int, default=1280, help="Width to resize video") + return parser + + +def init_t5(): + """Initialize and return the T5 tokenizer and text encoder.""" + tokenizer = T5TokenizerFast.from_pretrained("google-t5/t5-11b") + text_encoder = T5EncoderModel.from_pretrained("google-t5/t5-11b") + text_encoder.to("cuda") + text_encoder.eval() + return tokenizer, text_encoder + + +def init_video_tokenizer(tokenizer_dir: str): + """Initialize and return the Cosmos Video tokenizer.""" + dit_config = DiT7BConfig(vae_path=tokenizer_dir) + vae = dit_config.configure_vae() + return vae + + +@torch.no_grad() +def encode_for_batch(tokenizer, encoder, prompts: list[str], max_length=512): + """ + Encode a batch of text prompts to a batch of T5 embeddings. + Parameters: + tokenizer: T5 embedding tokenizer. + encoder: T5 embedding text encoder. + prompts: A batch of text prompts. + max_length: Sequence length of text embedding (defaults to 512). + """ + + batch_encoding = tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + # We expect all the processing is done on GPU. + input_ids = batch_encoding.input_ids.cuda() + attn_mask = batch_encoding.attention_mask.cuda() + + outputs = encoder(input_ids=input_ids, attention_mask=attn_mask) + encoded_text = outputs.last_hidden_state + + lengths = attn_mask.sum(dim=1).cpu() + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text + + +def create_condition_latent_from_input_frames(tokenizer, input_frames, num_frames_condition=25): + B, C, T, H, W = input_frames.shape + num_frames_encode = tokenizer.pixel_chunk_duration + assert input_frames.shape[2] >= num_frames_condition, ( + f"input_frames not enough for condition, require at least {num_frames_condition}, get {input_frames.shape[2]}, {input_frames.shape}" + ) + assert num_frames_encode >= num_frames_condition, ( + f"num_frames_encode should be larger than num_frames_condition, get {num_frames_encode}, {num_frames_condition}" + ) + + # Put the conditioal frames to the begining of the video, and pad the end with zero + condition_frames = input_frames[:, :, -num_frames_condition:] + padding_frames = condition_frames.new_zeros(B, C, num_frames_encode - num_frames_condition, H, W) + encode_input_frames = torch.cat([condition_frames, padding_frames], dim=2).to("cuda") + vae = tokenizer.to(encode_input_frames.device) + latent = vae.encode(encode_input_frames) + return latent, encode_input_frames + + +def main(args): + # Set up output directory + os.makedirs(args.output_path, exist_ok=True) + + # Initialize T5 + tokenizer, text_encoder = init_t5() + + # Initialize the VAE + if args.tokenizer_dir == "": + args.tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + vae = init_video_tokenizer(args.tokenizer_dir) + + # Constants + t5_embeding_max_length = 512 + chunk_duration = vae.video_vae.pixel_chunk_duration # Frames per chunk + cnt = 0 # File index + + # Check if dataset_path is correct + files = glob.glob(os.path.join(args.dataset_path, "*.mp4")) + if not files: + raise ValueError(f"Dataset path {args.dataset_path} does not contain any .mp4 files.") + + # Process each video in the dataset folder + with torch.no_grad(): + for video_path in tqdm(glob.glob(os.path.join(args.dataset_path, "*.mp4"))): + # Read video (T x H x W x C) + video, _, meta = torchvision.io.read_video(video_path) + T, H, W, C = video.shape + + # Skip videos shorter than one chunk + if T < chunk_duration: + log.info(f"Video {video_path} is shorter than {chunk_duration} frames. Skipped.") + continue + + # Sample random segments + for _ in range(args.num_chunks): + start_idx = random.randint(0, T - chunk_duration) + chunk = video[start_idx : start_idx + chunk_duration] # (chunk_duration, H, W, C) + + # Rearrange dimensions: (T, H, W, C) -> (T, C, H, W) + chunk = rearrange(chunk, "t h w c -> t c h w") + + # Resize to [704, 1280] for each frame + chunk = torchvision.transforms.functional.resize(chunk, [args.height, args.width]) + + # Expand dims: (T, C, H, W) -> (B=1, C, T, H, W) + chunk = rearrange(chunk, "(b t) c h w -> b c t h w", b=1) + + # Convert to bf16 and normalize from [0, 255] to [-1, 1] + chunk = chunk.to(device="cuda", dtype=torch.bfloat16, non_blocking=True) / 127.5 - 1.0 + + # Condition Latent (for Video2World training) + conditioning_chunk_len = 9 + conditioning_chunk = chunk[:, :, :conditioning_chunk_len, ...] + + # Encode video + latent = vae.encode(chunk).cpu() # shape: (1, latent_channels, T//factor, H//factor, W//factor) + + # Encode conditioning frames + conditioning_latent, _ = create_condition_latent_from_input_frames( + vae, conditioning_chunk, conditioning_chunk_len + ) + conditioning_latent = conditioning_latent.cpu() + + # Encode text + out = encode_for_batch(tokenizer, text_encoder, [args.prompt])[0] + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + + # Pad T5 embedding to t5_embeding_max_length + L, C_ = encoded_text.shape + t5_embed = torch.zeros(1, t5_embeding_max_length, C_, dtype=torch.bfloat16) + t5_embed[0, :L] = encoded_text + + # Save data to folder + torch.save(latent[0], os.path.join(args.output_path, f"{cnt}.video_latent.pth")) + torch.save(conditioning_latent[0], os.path.join(args.output_path, f"{cnt}.conditioning_latent.pth")) + torch.save(t5_embed[0], os.path.join(args.output_path, f"{cnt}.t5_text_embeddings.pth")) + + # Create a T5 text mask of all ones + torch.save( + torch.ones(512, dtype=torch.bfloat16), os.path.join(args.output_path, f"{cnt}.t5_text_mask.pth") + ) + + # Save metadata + info = { + "height": H, + "width": W, + "fps": meta["video_fps"], + "num_frames": chunk_duration, + "video_path": os.path.basename(video_path), + "start_frame": start_idx, + } + with open(os.path.join(args.output_path, f"{cnt}.info.json"), "w") as json_file: + json.dump(info, json_file) + + cnt += 1 + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/prepare_instruction_dataset.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/prepare_instruction_dataset.py new file mode 100644 index 00000000..668c1b7e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/prepare_instruction_dataset.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import glob +import json +import os +import random + +import torch +import torchvision +from cosmos1.utils import log +from einops import rearrange +from huggingface_hub import snapshot_download +from nemo.collections.diffusion.models.model import DiT7BConfig +from tqdm import tqdm +from transformers import T5EncoderModel, T5TokenizerFast + + +def get_parser(): + parser = argparse.ArgumentParser(description="Process some configurations.") + parser.add_argument("--tokenizer_dir", type=str, default="", help="Path to the VAE model") + parser.add_argument( + "--dataset_path", + type=str, + default="video_dataset", + help="Path to the dataset. A folder of with videos, instructions, and metas subfolders.", + ) + parser.add_argument("--output_path", type=str, default="video_dataset_cached", help="Path to the output directory") + parser.add_argument("--num_chunks", type=int, default=5, help="Number of random chunks to sample per video") + parser.add_argument("--height", type=int, default=704, help="Height to resize video") + parser.add_argument("--width", type=int, default=1280, help="Width to resize video") + return parser + + +def init_t5(): + """Initialize and return the T5 tokenizer and text encoder.""" + tokenizer = T5TokenizerFast.from_pretrained("google-t5/t5-11b") + text_encoder = T5EncoderModel.from_pretrained("google-t5/t5-11b") + text_encoder.to("cuda") + text_encoder.eval() + return tokenizer, text_encoder + + +def init_video_tokenizer(tokenizer_dir: str): + """Initialize and return the Cosmos Video tokenizer.""" + dit_config = DiT7BConfig(vae_path=tokenizer_dir) + vae = dit_config.configure_vae() + return vae + + +@torch.no_grad() +def encode_for_batch(tokenizer, encoder, prompts: list[str], max_length=512): + """ + Encode a batch of text prompts to a batch of T5 embeddings. + Parameters: + tokenizer: T5 embedding tokenizer. + encoder: T5 embedding text encoder. + prompts: A batch of text prompts. + max_length: Sequence length of text embedding (defaults to 512). + """ + + batch_encoding = tokenizer.batch_encode_plus( + prompts, + return_tensors="pt", + truncation=True, + padding="max_length", + max_length=max_length, + return_length=True, + return_offsets_mapping=False, + ) + + # We expect all the processing is done on GPU. + input_ids = batch_encoding.input_ids.cuda() + attn_mask = batch_encoding.attention_mask.cuda() + + outputs = encoder(input_ids=input_ids, attention_mask=attn_mask) + encoded_text = outputs.last_hidden_state + + lengths = attn_mask.sum(dim=1).cpu() + for batch_id in range(encoded_text.shape[0]): + encoded_text[batch_id][lengths[batch_id] :] = 0 + + return encoded_text + + +def create_condition_latent_from_input_frames(tokenizer, input_frames, num_frames_condition=25): + B, C, T, H, W = input_frames.shape + num_frames_encode = tokenizer.pixel_chunk_duration + assert input_frames.shape[2] >= num_frames_condition, ( + f"input_frames not enough for condition, require at least {num_frames_condition}, get {input_frames.shape[2]}, {input_frames.shape}" + ) + assert num_frames_encode >= num_frames_condition, ( + f"num_frames_encode should be larger than num_frames_condition, get {num_frames_encode}, {num_frames_condition}" + ) + + # Put the conditioal frames to the begining of the video, and pad the end with zero + condition_frames = input_frames[:, :, -num_frames_condition:] + padding_frames = condition_frames.new_zeros(B, C, num_frames_encode - num_frames_condition, H, W) + encode_input_frames = torch.cat([condition_frames, padding_frames], dim=2).to("cuda") + vae = tokenizer.to(encode_input_frames.device) + latent = vae.encode(encode_input_frames) + return latent, encode_input_frames + + +def main(args): + # Set up output directory + os.makedirs(args.output_path, exist_ok=True) + + # Initialize T5 + tokenizer, text_encoder = init_t5() + + # Initialize the VAE + if args.tokenizer_dir == "": + args.tokenizer_dir = snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8") + vae = init_video_tokenizer(args.tokenizer_dir) + + # Constants + t5_embeding_max_length = 512 + chunk_duration = vae.video_vae.pixel_chunk_duration # Frames per chunk + cnt = 0 # File index + + video_folder = os.path.join(args.dataset_path, "videos") + instruction_folder = os.path.join(args.dataset_path, "instructions") + + video_paths = glob.glob(os.path.join(video_folder, "*.mp4")) + # Check if dataset_path is correct + if not video_paths: + raise ValueError(f"Dataset path {args.dataset_path} does not contain any .mp4 files.") + + # Process each video in the dataset folder + with torch.no_grad(): + for video_path in tqdm(video_paths): + instruction_path = os.path.join(instruction_folder, os.path.basename(video_path).replace(".mp4", ".json")) + with open(instruction_path, "r") as f: + instruction = json.load(f)["language_instruction_0"] + # Read video (T x H x W x C) + video, _, meta = torchvision.io.read_video(video_path) + T, H, W, C = video.shape + + # Skip videos shorter than one chunk + if T < chunk_duration: + log.info(f"Video {video_path} is shorter than {chunk_duration} frames. Skipped.") + continue + + # Sample random segments + num_unique_chunks = T - chunk_duration + 1 + num_chunks = min(args.num_chunks, num_unique_chunks) + for ix in range(num_chunks): + if num_unique_chunks < args.num_chunks: + start_idx = ix + else: + start_idx = random.randint(0, T - chunk_duration) + chunk = video[start_idx : start_idx + chunk_duration] # (chunk_duration, H, W, C) + + # Rearrange dimensions: (T, H, W, C) -> (T, C, H, W) + chunk = rearrange(chunk, "t h w c -> t c h w") + + # Resize to [704, 1280] for each frame + chunk = torchvision.transforms.functional.resize(chunk, [args.height, args.width]) + + # Expand dims: (T, C, H, W) -> (B=1, C, T, H, W) + chunk = rearrange(chunk, "(b t) c h w -> b c t h w", b=1) + + # Convert to bf16 and normalize from [0, 255] to [-1, 1] + chunk = chunk.to(device="cuda", dtype=torch.bfloat16, non_blocking=True) / 127.5 - 1.0 + + # Condition Latent (for Video2World training) + conditioning_chunk_len = 9 + conditioning_chunk = chunk[:, :, :conditioning_chunk_len, ...] + + # Encode video + latent = vae.encode(chunk).cpu() # shape: (1, latent_channels, T//factor, H//factor, W//factor) + + # Encode conditioning frames + conditioning_latent, _ = create_condition_latent_from_input_frames( + vae, conditioning_chunk, conditioning_chunk_len + ) + conditioning_latent = conditioning_latent.cpu() + + # Encode text + out = encode_for_batch(tokenizer, text_encoder, [instruction])[0] + encoded_text = torch.tensor(out, dtype=torch.bfloat16) + + # Pad T5 embedding to t5_embeding_max_length + L, C_ = encoded_text.shape + t5_embed = torch.zeros(1, t5_embeding_max_length, C_, dtype=torch.bfloat16) + t5_embed[0, :L] = encoded_text + + # Save data to folder + torch.save(latent[0], os.path.join(args.output_path, f"{cnt}.video_latent.pth")) + torch.save(conditioning_latent[0], os.path.join(args.output_path, f"{cnt}.conditioning_latent.pth")) + torch.save(t5_embed[0], os.path.join(args.output_path, f"{cnt}.t5_text_embeddings.pth")) + + # Create a T5 text mask of all ones + torch.save( + torch.ones(512, dtype=torch.bfloat16), os.path.join(args.output_path, f"{cnt}.t5_text_mask.pth") + ) + + # Save metadata + info = { + "height": H, + "width": W, + "fps": meta["video_fps"], + "num_frames": chunk_duration, + "video_path": os.path.basename(video_path), + "start_frame": start_idx, + } + with open(os.path.join(args.output_path, f"{cnt}.info.json"), "w") as json_file: + json.dump(info, json_file) + print(f"Saved metadata to {args.output_path}/{cnt}.info.json") + + cnt += 1 + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/torch/megatron/megatron-gpt-345m_merges b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/torch/megatron/megatron-gpt-345m_merges new file mode 100644 index 00000000..226b0752 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/torch/megatron/megatron-gpt-345m_merges @@ -0,0 +1,50001 @@ +#version: 0.2 +Ġ t +Ġ a +h e +i n +r e +o n +Ġt he +e r +Ġ s +a t +Ġ w +Ġ o +e n +Ġ c +i t +i s +a n +o r +e s +Ġ b +e d +Ġ f +in g +Ġ p +o u +Ġa n +a l +a r +Ġt o +Ġ m +Ġo f +Ġ in +Ġ d +Ġ h +Ġan d +i c +a s +l e +Ġt h +i on +o m +l l +en t +Ġ n +Ġ l +s t +Ġ re +v e +Ġ e +r o +l y +Ġb e +Ġ g +Ġ T +c t +Ġ S +i d +o t +Ġ I +u t +e t +Ġ A +Ġ is +Ġ on +i m +a m +o w +a y +a d +s e +Ġth at +Ġ C +i g +Ġf or +a c +Ġ y +v er +u r +Ġ u +l d +Ġs t +Ġ M +' s +Ġ he +Ġ it +at ion +it h +i r +c e +Ġy ou +i l +Ġ B +Ġw h +o l +Ġ P +Ġw ith +Ġ 1 +t er +c h +Ġa s +Ġw e +Ġ ( +n d +i ll +Ġ D +i f +Ġ 2 +a g +er s +k e +Ġ " +Ġ H +e m +Ġc on +Ġ W +Ġ R +he r +Ġw as +Ġ r +o d +Ġ F +u l +at e +Ġa t +r i +p p +o re +ĠT he +Ġs e +u s +Ġp ro +Ġh a +u m +Ġa re +Ġd e +a in +an d +Ġo r +ig h +es t +is t +a b +r om +Ġ N +t h +Ġc om +Ġ G +u n +o p +0 0 +Ġ L +Ġn ot +es s +Ġe x +Ġ v +re s +Ġ E +e w +it y +an t +Ġb y +e l +o s +or t +o c +q u +Ġf rom +Ġha ve +Ġs u +i ve +ou ld +Ġs h +Ġth is +n t +r a +p e +igh t +ar t +m ent +Ġa l +u st +en d +- - +al l +Ġ O +ac k +Ġc h +Ġ le +i es +re d +ar d +â Ģ +ou t +Ġ J +Ġa b +e ar +i v +al ly +ou r +o st +g h +p t +Ġp l +as t +Ġc an +a k +om e +u d +T he +Ġh is +Ġd o +Ġg o +Ġh as +g e +' t +Ġ U +r ou +Ġs a +Ġ j +Ġb ut +Ġw or +Ġa ll +e ct +Ġ k +am e +Ġw ill +o k +Ġw he +Ġthe y +id e +0 1 +f f +ic h +p l +t her +Ġt r +. . +Ġin t +i e +u re +ag e +Ġn e +i al +a p +in e +ic e +Ġm e +Ġo ut +an s +on e +on g +ion s +Ġwh o +Ġ K +Ġu p +Ġthe ir +Ġa d +Ġ 3 +Ġu s +at ed +ou s +Ġm ore +u e +o g +ĠS t +in d +i ke +Ġs o +im e +p er +. " +b er +i z +a ct +Ġon e +Ġsa id +Ġ - +a re +Ġyou r +c c +ĠT h +Ġc l +e p +a ke +ab le +i p +Ġcon t +Ġwh ich +i a +Ġ im +Ġab out +Ġwe re +ver y +u b +Ġh ad +Ġ en +Ġcom p +, " +ĠI n +Ġu n +Ġa g +i re +ac e +a u +ar y +Ġw ould +as s +r y +Ġ âĢ +c l +o ok +e re +s o +Ġ V +ig n +i b +Ġof f +Ġt e +v en +Ġ Y +i le +o se +it e +or m +Ġ2 01 +Ġre s +Ġm an +Ġp er +Ġo ther +or d +ul t +Ġbe en +Ġl ike +as e +an ce +k s +ay s +ow n +en ce +Ġd is +ct ion +Ġan y +Ġa pp +Ġs p +in t +res s +ation s +a il +Ġ 4 +ic al +Ġthe m +Ġhe r +ou nt +ĠC h +Ġa r +Ġ if +Ġthe re +Ġp e +Ġy ear +a v +Ġm y +Ġs ome +Ġwhe n +ou gh +ac h +Ġth an +r u +on d +ic k +Ġo ver +ve l +Ġ qu +Ċ Ċ +Ġs c +re at +re e +ĠI t +ou nd +p ort +Ġal so +Ġp art +f ter +Ġk n +Ġbe c +Ġt ime +en s +Ġ 5 +op le +Ġwh at +Ġn o +d u +m er +an g +Ġn ew +-- -- +Ġg et +or y +it ion +ing s +Ġj ust +Ġint o +Ġ 0 +ent s +o ve +t e +Ġpe ople +Ġp re +Ġit s +Ġre c +Ġt w +i an +ir st +ar k +or s +Ġwor k +ad e +o b +Ġs he +Ġo ur +w n +in k +l ic +Ġ1 9 +ĠH e +is h +nd er +au se +Ġh im +on s +Ġ [ +Ġ ro +f orm +i ld +at es +ver s +Ġon ly +o ll +Ġs pe +c k +e ll +am p +Ġa cc +Ġb l +i ous +ur n +f t +o od +Ġh ow +he d +Ġ ' +Ġa fter +a w +Ġat t +o v +n e +Ġpl ay +er v +ic t +Ġc ould +it t +Ġa m +Ġf irst +Ġ 6 +Ġa ct +Ġ $ +e c +h ing +u al +u ll +Ġcom m +o y +o ld +c es +at er +Ġf e +Ġbe t +w e +if f +Ġtw o +oc k +Ġb ack +) . +id ent +Ġu nder +rou gh +se l +x t +Ġm ay +rou nd +Ġp o +p h +is s +Ġd es +Ġm ost +Ġd id +Ġad d +j ect +Ġin c +f ore +Ġp ol +on t +Ġag ain +cl ud +ter n +Ġkn ow +Ġne ed +Ġcon s +Ġc o +Ġ . +Ġw ant +Ġse e +Ġ 7 +n ing +i ew +ĠTh is +c ed +Ġe ven +Ġin d +t y +ĠW e +at h +Ġthe se +Ġp r +Ġu se +Ġbec ause +Ġf l +n g +Ġn ow +ĠâĢ ĵ +c om +is e +Ġm ake +Ġthe n +ow er +Ġe very +ĠU n +Ġse c +os s +u ch +Ġe m +Ġ = +ĠR e +i ed +r it +Ġin v +le ct +Ġsu pp +at ing +Ġl ook +m an +pe ct +Ġ 8 +ro w +Ġb u +Ġwhe re +if ic +Ġyear s +i ly +Ġd iff +Ġsh ould +Ġre m +T h +I n +Ġe v +d ay +' re +ri b +Ġre l +s s +Ġde f +Ġr ight +Ġs y +) , +l es +00 0 +he n +Ġth rough +ĠT r +_ _ +Ġw ay +Ġd on +Ġ , +Ġ1 0 +as ed +Ġas s +ub lic +Ġre g +ĠA nd +i x +Ġ very +Ġin clud +ot her +Ġim p +ot h +Ġsu b +ĠâĢ Ķ +Ġbe ing +ar g +ĠW h += = +ib le +Ġdo es +an ge +r am +Ġ 9 +er t +p s +it ed +ation al +Ġb r +Ġd own +Ġman y +ak ing +Ġc all +ur ing +it ies +Ġp h +ic s +al s +Ġde c +at ive +en er +Ġbe fore +il ity +Ġwe ll +Ġm uch +ers on +Ġth ose +Ġsu ch +Ġ ke +Ġ end +ĠB ut +as on +t ing +Ġl ong +e f +Ġth ink +y s +Ġbe l +Ġs m +it s +a x +Ġo wn +Ġpro v +Ġs et +if e +ment s +b le +w ard +Ġsh ow +Ġp res +m s +om et +Ġo b +Ġs ay +ĠS h +t s +f ul +Ġe ff +Ġg u +Ġin st +u nd +re n +c ess +Ġ ent +ĠY ou +Ġgo od +Ġst art +in ce +Ġm ade +t t +st em +ol og +u p +Ġ | +um p +Ġhe l +ver n +ul ar +u ally +Ġa c +Ġm on +Ġl ast +Ġ2 00 +1 0 +Ġst ud +u res +ĠA r +sel f +ar s +mer ic +u es +c y +Ġm in +oll ow +Ġc ol +i o +Ġm od +Ġc ount +ĠC om +he s +Ġf in +a ir +i er +âĢ Ķ +re ad +an k +at ch +e ver +Ġst r +Ġpo int +or k +ĠN ew +Ġs ur +o ol +al k +em ent +Ġus ed +ra ct +we en +Ġs ame +ou n +ĠA l +c i +Ġdiff ere +Ġwh ile +---- ---- +Ġg ame +ce pt +Ġs im +.. . +Ġin ter +e k +Ġre port +Ġpro du +Ġst ill +l ed +a h +Ġhe re +Ġwor ld +Ġth ough +Ġn um +ar ch +im es +al e +ĠS e +ĠI f +/ / +ĠL e +Ġre t +Ġre f +Ġtr ans +n er +ut ion +ter s +Ġt ake +ĠC l +Ġcon f +w ay +a ve +Ġgo ing +Ġs l +u g +ĠA meric +Ġspe c +Ġh and +Ġbet ween +ist s +ĠD e +o ot +I t +Ġe ar +Ġagain st +Ġh igh +g an +a z +at her +Ġex p +Ġo p +Ġin s +Ġg r +Ġhel p +Ġre qu +et s +in s +ĠP ro +is m +Ġf ound +l and +at a +us s +am es +Ġp erson +Ġg reat +p r +Ġs ign +ĠA n +' ve +Ġs omet +Ġs er +h ip +Ġr un +Ġ : +Ġt er +ire ct +Ġf ollow +Ġd et +ic es +Ġf ind +1 2 +Ġm em +Ġc r +e red +e x +Ġex t +ut h +en se +c o +Ġte am +v ing +ou se +as h +at t +v ed +Ġsy stem +ĠA s +d er +iv es +m in +Ġle ad +ĠB l +c ent +Ġa round +Ġgo vern +Ġc ur +vel op +an y +Ġc our +al th +ag es +iz e +Ġc ar +od e +Ġl aw +Ġre ad +' m +c on +Ġre al +Ġsupp ort +Ġ1 2 +.. .. +Ġre ally +n ess +Ġf act +Ġd ay +Ġb oth +y ing +Ġs erv +ĠF or +Ġth ree +Ġw om +Ġm ed +od y +ĠThe y +5 0 +Ġex per +t on +Ġe ach +ak es +Ġc he +Ġc re +in es +Ġre p +1 9 +g g +ill ion +Ġg rou +ut e +i k +W e +g et +E R +Ġm et +Ġs ays +o x +Ġd uring +er n +iz ed +a red +Ġf am +ic ally +Ġha pp +ĠI s +Ġch ar +m ed +v ent +Ġg ener +i ent +p le +i et +re nt +1 1 +v es +pt ion +Ġ2 0 +form ation +Ġc or +Ġoff ic +ie ld +Ġto o +is ion +Ġin f +Ġ Z +t he +o ad +Ġp ublic +Ġpro g +r ic +* * +Ġw ar +Ġp ower +v iew +Ġf ew +Ġl oc +Ġdiffere nt +Ġst ate +Ġhe ad +' ll +Ġp oss +Ġst at +re t +ant s +Ġv al +Ġis s +Ġc le +i vers +an c +Ġex pl +Ġan other +Ġ Q +Ġa v +th ing +n ce +W h +Ġch ild +Ġs ince +i red +l ess +Ġl ife +Ġde velop +itt le +Ġde p +Ġp ass +ã ĥ +Ġt urn +or n +Th is +b ers +ro ss +ĠA d +Ġf r +Ġres p +Ġsec ond +o h +Ġ / +Ġdis c +Ġ & +Ġsomet hing +Ġcomp le +Ġ ed +Ġf il +Ġmon th +a j +u c +Ġgovern ment +Ġwith out +Ġle g +Ġd ist +Ġp ut +Ġqu est +an n +Ġpro t +2 0 +Ġne ver +i ence +Ġle vel +Ġar t +Ġth ings +Ġm ight +Ġeff ect +Ġcont ro +Ġc ent +Ġ1 8 +Ġall ow +Ġbel ie +ch ool +ot t +Ġinc re +Ġfe el +Ġres ult +Ġl ot +Ġf un +ot e +Ġt y +ere st +Ġcont in +Ġus ing +Ġb ig +2 01 +Ġas k +Ġb est +Ġ ) +I N +Ġo pp +3 0 +Ġnum ber +in ess +S t +le ase +Ġc a +Ġm ust +Ġd irect +Ġg l +Ġ < +Ġop en +Ġp ost +Ġcom e +Ġse em +ord ing +Ġwe ek +ate ly +it al +Ġe l +ri end +Ġf ar +Ġt ra +in al +Ġp ri +ĠU S +Ġpl ace +Ġfor m +Ġto ld +" : +ain s +at ure +ĠTr ump +Ġst and +Ġ # +id er +ĠF r +Ġne xt +Ġs oc +Ġp ur +Ġle t +Ġl ittle +Ġh um +Ġ i +r on +1 5 +Ġ1 5 +Ġcomm un +Ġm ark +ĠThe re +Ġw r +ĠTh at +Ġin formation +w ays +Ġb us +a pp +Ġinv est +m e +Ġh ard +ain ed +e ad +Ġim port +Ġapp ro +Ġt est +Ġt ri +Ġre st +os ed +Ġf ull +Ġc are +ĠS p +Ġc ase +O N +Ġs k +Ġl ess +Ġ + +Ġpart ic +ĠP l +ab ly +u ck +is hed +ch n +b e +Ġl ist +at or +Ġto p +Ġad v +ĠB e +ru ct +Ġd em +r ation +l ing +g y +re en +g er +Ġh ome +Ġle ft +Ġbet ter +Ġd ata +Ġ1 1 +Ġatt ack +Ġpro ble +l ine +ard s +Ġbe h +r al +ĠH ow +ĠS he +ar ge +Ġ -- +: // +Ġb ro +ĠP h +at s +Ġbu ild +w w +id ed +a im +as es +en cy +Ġm ain +in ed +Ġinclud ing +Ġ { +Ġg ot +Ġint erest +Ġke ep +Ġ X +Ġe as +ain ing +Ġcl ass +âĢ ¦ +ĠN o +Ġv ar +Ġsm all +amp le +A T +Ġ ide +ĠS o +Ġre ce +Ġpol it +Ġm ov +Ġpl an +Ġper cent +iv ing +Ġc amp +Ġp ay +1 4 +s c +is ed +Ġu nt +one y +pl oy +== == +Ġdid n +ĠI nd +el s +ert ain +Ġp os +__ __ +i ver +Ġpro cess +Ġprog ram +if ied +ĠR ep +1 6 +u ro +olog y +at ter +in a +Ġn ame +ĠA ll +Ġf our +Ġret urn +v ious +b s +Ġcall ed +Ġm ove +ĠS c +ir d +Ġgrou p +Ġb re +Ġm en +Ġc ap +t en +e e +Ġd ri +le g +he re +uth or +Ġp at +Ġcur rent +id es +Ġp op +t o +ent ion +Ġal ways +Ġm il +Ġwom en +Ġ1 6 +Ġo ld +iv en +ra ph +ĠO r +r or +ent ly +Ġn ear +ĠE x +re am +s h +Ġ1 4 +Ġf ree +iss ion +st and +ĠC on +al ity +us ed +1 3 +Ġdes ign +Ġch ange +Ġch ang +Ġb o +Ġv is +em ber +Ġb ook +read y +Ġk ill +2 5 +pp ed +Ġa way +Ġab le +Ġcount ry +Ġcon st +ar n +Ġor der +A R +i or +i um +or th +1 8 +ail able +Ġs w +Ġm illion +Ġ1 3 +at ic +t ed +ĠG o +Ġo per +en g +Ġth ing +aj or +con om +ĠCom m +Ġwh y +u red +ur al +Ġs chool +b y +ĠM ar +Ġa ff +Ġd ays +Ġan n +us h +an e +I f +e g +Ġpro f +Ġhe alth +ou th +B ut +ion al +. , +Ġs ol +Ġal ready +Ġ3 0 +Ġchar act +H e +Ġf riend +E S +i ans +ic le +' d +ĠO n +Ġle ast +Ġp rom +Ġd r +Ġh ist +it her +Ġ est +i qu +1 7 +s on +Ġte ll +Ġt alk +oh n +o int +le ction +A N +Ġunt il +au gh +Ġl ater +Ġ ve +Ġv iew +end ing +iv ed +Ġwor d +w are +Ġc ost +Ġen ough +Ġg ive +ĠUn ited +Ġte chn +are nt +O R +Ġp ar +ĠD r +Ġ201 6 +r ist +er ing +Ġ  +Ġl arge +s ide +ac y +cc ess +Ġw in +Ġimport ant +Ġ19 9 +Ġdoes n +Ġ1 7 +Ġbus iness +Ġcle ar +Ġre se +" , +ur y +Ġe qu +as ter +al f +ĠAmeric an +n ect +Ġex pect +ivers ity +Ġo cc +ĠF l +Ġk ind +Ġme an +Ġp ast +Ġde v +Ġb as +le t +ra ft +Ġor gan +Ġde l +Ġper form +Ġst ory +Ġse ason +ĠC ol +Ġcl aim +Ġc ame +Ġwith in +Ġl ine +Ġpro ject +ĠA t +Ġcontro l +end ed +ĠS y +Ġa ir +iz ation +Ġ * +le y +Ġm oney +id d +Y ou +f or +Ġfam ily +Ġm aking +Ġb it +Ġpol ice +Ġhapp en +Ġ vers +on y +u ff +ĠW hen +Ġs it +ide o +l f +is on +Ġsu re +g in +Ġapp ear +Ġl ight +Ġ es +o f +Ġw ater +Ġt imes +n ot +Ġg row +Ġcomp any +ĠT e +ow s +Ġm ar +our ce +i ol +ar m +b r +Ġex ample +Ġcon c +Ġf ore +ĠT o +p ro +E N +ri es +Ġ2 5 +ĠC an +ne y +Ġact ually +Ġe ver +ur ity +ak en +ap s +Ġt ax +Ġm ajor +am a +Ġof ten +er al +Ġhum an +Ġj ob +is ter +Ġav ailable +oc r +en n +a id +iv id +Ġrec ord +? " +Ġs ing +ĠA m +id ence +Ġnew s +st er +Ġe conom +Ġfollow ing +ĠB r +is ing +Ġh our +m ost +um ent +Ġse x +Ġdes c +Ġbec ome +ĠE d +Ġto ok +Ġha ving +Ġprodu ct +a ult +A s +ar ing +Ġme ans +Ġh op +un e +Ġch o +Ġc ertain +Ġn on +Ġde al +2 4 +le ment +oc i +en e +Ġs ide +ĠP r +ĠM ay +Ġre ason +u ed +c hed +ul ation +Ġe lect +Ġoffic ial +Ġposs ible +Ġh old +and s +ot s +Ġc ity +or ies +Ġse ver +Ġchild ren +Ġon ce +Ġact iv +l er +Ġn ight +it ions +ĠJ ohn +a pe +pl ay +Ġd one +Ġl im +Ġwork ing +ĠP res +or ld +e b +ĠC o +Ġb ody +ail s +ut es +ĠM r +Ġwhe ther +Ġa uthor +ro p +Ġpro per +Ġse en +) ; +Ġf ac +ĠS u +Ġcon d +it ing +Ġcour se +Ġ } +-------- -------- +a ign +Ġev ent +Ġen g +Ġp ot +Ġin tern +i am +Ġsh ort +em pt +ã Ĥ +ĠG od +il ar +8 0 +Ġor ig +I S +our n +ab ility +it ive +Ġd am +Ġ1 00 +Ġp ress +Ġdo ing +Ġprot ect +r ing +Ġthough t +Ġquest ion +re w +ĠW ar +Ġsever al +ĠSt ate +Ġg iven +Ġf und +ĠT w +Ġw ent +an ces +w ork +p or +m y +4 0 +Ġar g +art ment +ust om +Ġpol ic +Ġme et +Ġc reat +2 2 +ĠSt ates +Ġg ames +ra w +ut ure +Ġunder stand +ur s +ĠO b +l ish +s y +Ġm akes +Ġw on +ag on +Ġh tt +Ġl ove +ent ial +Ġcomple te +p ar +ĠI m +A L +Ġacc ount + ł +ore d +ver t +Ġ ident +Ġ201 5 +Ġother s +ĠM in +i ber +ver age +The re +ition al +d d +Ġpro b +Ġyou ng +Ġal ong +Ġacc ording +Ġy et +Ġmem bers +ĠWh at +o id +ĠM an +A nd +Ġam ong +a i +Ġem ploy +ĠR es +Ġ > +Ġinv ol +Ġl ow +a f +ĠC ar +Ġh ig +ĠO ne +ĠS ec +in ation +Ġlike ly +Ġan t +ag ed +ĠR uss +Ġb en +Ġre le +F or +b ack +ĠN ot +Ġpres ident +b all +Ġacc ess +ivid ual +ĠD em +ĠE uro +6 0 +Ġkn own +ir l +ĠG r +Ġear ly +u se +iet y +âĢ ĵ +Ġf ight +Ġs ent +Ġto day +Ġmark et +" . +Ġb ased +Ġstr ong +ur ther +Ġde b +m ber +Ġproble m +Ġde ath +Ġsoc ial +im ate +A S +ort un +Ġcamp aign +er y +C h +Ġe y +i ally +Ġm us +w h +p os +Ġ er +Ġsa f +Ġmonth s +ir on +Ġv iol +Ġf ive +Ġst re +Ġplay ers +in c +al d +y ear +a un +Ġsu ccess +Ġpres ent +ere nce +Ġ201 4 +Ġsu gg +Ġpartic ular +Ġtr y +Ġsugg est +ĠCh rist +on es +Ġpri v +2 3 +Ġc rit +Ġl and +Ġloc al +if y +2 9 +Ġa ut +E D +ĠG u +Ġm ult +Ġpolit ical +Ġask ed +Ġfor mer +it ter +ri pt +Ġcl ose +Ġp ract +ĠY ork +Ġget ting +Ġac ross +Ġcom b +Ġbelie ve +Ġ z +Ġto get +Ġtoget her +ĠC ent +ir c +Ġind ividual +ĠM c +2 7 +is k +ĠE ng +Ġf ace +Ġ2 4 +Ġval ue +Ġare a +e v +Ġw rit +ĠPres ident +Ġv ot +Ġke y +Ġm om +p ut +Ġany thing +Ġexper ience +att le +Ġm ind +a ff +om m +Ġf uture +g ed +Ġc ut +Ġto t +it ch +Ġv ideo +Ġinvest ig +Ġn et +ĠM y +r ict +i en +. ) +Ġimp ro +th ough +ward s +Ġcon nect +ĠM ed +sel ves +ens ive +m b +o ber +at ors +A n +Ġ5 0 +Ġre du +res ent +Ġab ove +Ġf re +ĠEuro pe +s w +Ġam ount +ĠA pp +Ġe ither +Ġmil it +Ġan al +Ġf ail +ĠE n +al es +Ġspec ial +Ġbl ack +I T +c her +Ġlook ing +Ġf ire +y n +Ġal most +o on +Ġstud y +Ġm iss +c hes +ro wn +Ġt re +Ġcommun ity +Ġmed ia +Ġf ood +Ġcom es +ĠUn iversity +Ġsing le +Wh at +u ly +Ġh alf +ag ue +h od +ĠRep ublic +Ġstart ed +Ġqu ick +ot o +b ook +Ġiss ue +it or +Ġel se +Ġcons ider +2 6 +ro du +Ġt aken +2 8 +9 9 +ĠW ith +Ġtr ue +Ġw a +Ġtr ad +Ġag o +Ġm ess +ie f +Ġadd ed +o ke +Ġb ad +Ġf av +3 3 +Ġsim ilar +as k +ĠD on +Ġcharact er +ort s +ĠH ouse +Ġreport ed +Ġty pe +v al +i od +ĠHow ever +Ġt arg +Ġent ire +pp ing +Ġhist ory +Ġl ive +ff ic +.... .... +ed eral +Ġtr ying +Ġdisc uss +ĠH ar +ac es +l ished +Ġse lf +os p +re st +Ġro om +el t +Ġf all +ol ution +Ġe t +Ġ x +Ġis n +Ġide a +b o +Ġs ound +ĠD ep +Ġsome one +ci ally +ull y +Ġf oc +Ġob ject +if t +ap er +Ġplay er +Ġr ather +Ġserv ice +as hing +ĠD o +ĠP art +ru g +m on +p ly +Ġm or +Ġnot hing +Ġprov ide +I C +un g +Ġpart y +Ġex ist +Ġm ag +7 0 +Ġr ul +Ġh ouse +Ġbeh ind +Ġhow ever +ĠW orld +Ġs um +Ġapp lic +Ġ ; +Ġfun ction +g r +ĠP ol +Ġfr ont +2 00 +Ġser ies +Ġt em +Ġty p +ill s +Ġo pt +Ġpoint s +Ġbel ow +itt ed +Ġspec ific +Ġ201 7 +um b +Ġr a +Ġpre vious +Ġpre t +re me +Ġc ustom +Ġcour t +ĠM e +Ġre pl +Ġwho le +g o +c er +Ġt reat +ĠA ct +Ġprob ably +Ġle arn +end er +ĠA ss +Ġvers ion +n ow +Ġche ck +ĠC al +R E +min ist +O n +our ces +Ġben ef +Ġd oc +Ġdet er +Ġen c +Ġsu per +Ġadd ress +Ġv ict +Ġ201 3 +Ġme as +t r +Ġf ield +W hen +Ġsign ific +u ge +Ġfe at +Ġcomm on +l oad +Ġbe gin +Ġbr ing +Ġa ction +er man +Ġdesc rib +Ġind ust +Ġwant ed +ri ed +m ing +Ġatt empt +4 5 +f er +Ġd ue +ress ion +# # +Ġsh all +Ġs ix +o o +Ġst ep +Ġp ub +Ġhim self +Ġ2 3 +Ġc op +Ġd est +Ġst op +A C +ib ility +Ġl ab +ic ult +Ġhour s +Ġcre ate +Ġf urther +ĠAmeric a +ĠC ity +Ġd ou +he ad +S T +ĠN orth +c ing +Ġn ational +u le +ĠIn st +Ġt aking +ĠQ u +ir t +Ġre d +Ġrese arch +v iron +ĠG e +Ġbre ak +an a +Ġsp ace +ater ial +Ġrec ent +ĠA b +Ġgener al +Ġh it +Ġper iod +Ġevery thing +ive ly +Ġph ys +Ġsay ing +an ks +Ġc ou +Ġc ult +ac ed +e al +u ation +Ġc oun +l u +Ġinclud e +Ġpos ition +ĠA fter +ĠCan ad +ĠE m +Ġim m +ĠR ed +Ġp ick +Ġcom pl +Ġm atter +re g +e xt +ang u +is c +o le +a ut +Ġcomp et +e ed +f ect +Ġ2 1 +ĠS en +ĠThe se +as ing +Ġcan not +Ġin it +Ġrel ations +ac hed +Ġb ar +Ġ4 0 +ĠT H +Ġ201 2 +Ġv ol +Ġg round +Ġsec urity +Ġup d +il t +3 5 +Ġconc ern +ĠJ ust +Ġwh ite +Ġseem s +ĠH er +pe cially +i ents +Ġann oun +Ġf ig +ight s +Ġst ri +l ike +id s +Ġs us +Ġw atch +Ġ â +Ġw ind +ĠC ont +Ġit self +Ġm ass +A l +y le +iqu e +ĠN ational +Ġab s +Ġp ack +Ġout side +Ġan im +Ġp ain +et er +Ġman ag +du ct +og n +Ġ ] +ĠSe pt +se c +o ff +ĠJ an +Ġf oot +ad es +Ġth ird +Ġm ot +Ġev idence +int on +Ġth reat +a pt +pl es +c le +Ġl o +Ġde cl +Ġit em +med i +Ġrep resent +om b +am er +Ġsignific ant +og raph +s u +Ġc al +i res +00 00 +I D +A M +Ġsim ply +Ġlong er +Ġf ile +O T +c he +S o +ate g +or g +ĠH is +Ġen er +Ġd om +Ġup on +il i +": " +Ġthem selves +Ġcom ing +Ġqu ite +Ġdiff icult +ĠB ar +il ities +re l +end s +c ial +6 4 +Ġwom an +ra p +y r +Ġne cess +ip s +Ġte xt +Ġrequ ire +Ġmilit ary +Ġre view +Ġresp ons +7 5 +Ġsub ject +Ġinst ead +Ġiss ues +Ġg en +" ," +Ġmin utes +Ġwe ap +r ay +am ed +t ime +b l +H ow +Ġc ode +ĠS m +Ġhig her +ĠSt e +r is +Ġp age +Ġstud ents +ĠIn tern +Ġmet hod +ĠA ug +ĠP er +ĠA g +Ġpolic y +ĠS w +Ġex ec +Ġac cept +um e +rib ut +Ġword s +Ġfin al +Ġchang es +ĠDem ocr +Ġfriend s +Ġres pect +Ġe p +Ġcomp an +iv il +Ġdam age +** ** +og le +viron ment +Ġne g +ent al +Ġa p +Ġtot al +iv al +! " +l im +Ġneed s +Ġag re +Ġdevelop ment +Ġa ge +ip le +2 1 +Ġresult s +ĠA f +S h +Ġg un +ĠOb ama +ro ll +Ġ @ +Ġright s +ĠB rit +Ġrun ning +Ġwas n +Ġp ort +Ġr ate +Ġpret ty +Ġtarg et +Ġsa w +Ġc irc +Ġwor ks +ic ro +al t +o ver +ww w +Th at +l ier +Ġevery one +ud e +Ġp ie +idd le +ra el +Ġr ad +Ġbl ock +Ġw alk +T o +ã ģ +n es +ĠA ust +a ul +ro te +ĠS outh +ess ion +op h +Ġshow s +Ġs ite +Ġj o +Ġr isk +cl us +l t +Ġin j +id ing +ĠS pe +Ġch all +ir m +Ġ2 2 +itt ing +st r +Ġh y +L E +ke y +Ġbe gan +at ur +ashing ton +l am +ĠD av +b it +Ġs ize +ĠP ar +3 8 +ourn al +f ace +Ġdec ision +Ġl arg +Ġj ud +re ct +Ġcontin ue +ĠO ct +ove red +ĠI nt +==== ==== +Ġp arent +ĠW ill +Ġeas y +Ġd rug +ang er +Ġs ense +Ġd i +id ay +Ġener gy +ist ic +Ġass oci +ar ter +ob al +e ks +ĠE l +ur ch +Ġg irl +o e +it le +Ġ2 8 +ĠC he +Ġrequ est +Ġso on +Ġh ost +k y +Ġst ates +om es +Ġm aterial +le x +Ġmom ent +Ġan sw +on se +Ġes pecially +Ġn orm +Ġserv ices +p ite +r an +Ġro le +4 4 +) : +Ġc red +C l +____ ____ +Ġm at +Ġl og +ĠCl inton +O U +Ġoff ice +Ġ2 6 +Ġch arg +Ġtr ack +m a +Ġhe art +Ġb all +Ġperson al +Ġbuild ing +n a +s et +b ody +ĠBl ack +Ġincre ase +itt en +Ġneed ed +3 6 +3 2 += " +Ġl ost +Ġbec ame +Ġgrou ps +ĠM us +Ġw rote +ĠP e +Ġpro p +j oy +à © +ĠWh ite +Ġde ad +. ' +Ġhtt p +Ġwe bs +O S +Ġins ide +Ġwr ong +Ġstat ement +Ġ ... +y l +Ġfil m +Ġmus ic +Ġsh are +ific ation +Ġre lease +Ġfor ward +Ġst ay +Ġcomp ut +it te +s er +Ġorig inal +Ġc ard +Ġc and +Ġd iv +at ural +Ġfav or +O M +Ġc ases +us es +Ġse ction +Ġle ave +g ing +ov ed +ĠW ashington +3 9 +ĠG l +Ġrequ ired +act ion +ap an +o or +it er +ĠK ing +Ġcount ries +ĠG erman +ll ing +Ġ2 7 +3 4 +Ġquest ions +Ġpr im +Ġc ell +Ġsh oot +Ġany one +ĠW est +Ġaff ect +ep end +Ġon line +ĠIs rael +ĠSept ember +Ġab ility +Ġcont ent +is es +Ġre ve +Ġl aun +Ġind ic +Ġfor ce +c ast +Ġso ld +av ing +f l +Ġso ft +Ġcompan ies +ce ed +Ġart icle +Ġa ud +Ġre v +Ġed uc +Ġplay ing +0 5 +Ġhe ld +ct or +Ġrele ased +Ġf ederal +3 7 +Ġad minist +Ġinter view +Ġinst all +Ġrece ived +Ġs ource +u k +P h +Ġser ious +Ġcre ated +Ġc ause +Ġim medi +Ġdef in +u el +ĠDep artment +ct ions +ĠC our +ĠN ow +z e +it es +it ution +Ġl ate +Ġspe ak +n ers +Ġleg al +ar i +ĠC or +Ġwe eks +Ġmod el +Ġp red +Ġex act +B C +ĠB y +IN G +os ing +Ġt akes +Ġreg ard +Ġopp ortun +Ġpr ice +Ġ19 8 +ĠA pr +f ully +Ġor d +Ġproble ms +ru ction +h am +ĠC ount +le ge +Ġlead ers +E T +le v +Ġde ep +olog ical +es e +h aps +ĠS ome +Ġp ers +Ġcont ract +Ġrelations hip +s p +ou d +Ġb ase +4 8 +m it +A d +anc ial +Ġcons um +Ġpot ential +Ġl angu +re m +et h +Ġrel ig +ress ed +6 6 +Ġl ink +Ġl ower +ay er +ĠJ une +Ġf em +un t +er c +ur d +Ġcont act +Ġ ill +Ġm other +Ġest ab +h tt +ĠM arch +ĠB ro +ĠCh ina +Ġ2 9 +Ġs qu +Ġprov ided +Ġa verage +as ons +Ġ201 1 +Ġex am +l in +5 5 +n ed +Ġper fect +Ġt ou +al se +u x +Ġbu y +Ġsh ot +Ġcol lect +Ġph ot +Ġplay ed +Ġsur pr +Ġofficial s +Ġsim ple +av y +Ġindust ry +Ġhand s +g round +Ġp ull +Ġr ound +Ġus er +Ġr ange +u ary +Ġpriv ate +op s +e es +Ġw ays +ĠM ich +Ġve h +Ġex cept +Ġter ms +im um +pp er +I ON +ore s +ĠDr agon +ou l +Ġd en +Ġperform ance +Ġb ill +c il +4 7 +Ġen vironment +Ġex c +ad d +Ġwor th +Ġp ict +Ġch ance +Ġ201 8 +b or +Ġspe ed +ict ion +Ġal leg +ĠJ apan +at ory +re et +Ġm atch +ĠI I +Ġst ru +ord er +Ġst e +Ġl iving +Ġst ruct +in o +Ġse par +her n +Ġresp onse +Ġen joy +Ġv ia +A D +um ents +ace book +Ġmem ber +ib r +iz ing +Ġto ol +ĠM on +ĠWh ile +h ood +ĠA ng +ĠD ef +Ġoff er +T r +a ur +Ġturn ed +ĠJ uly +d own +an ced +Ġrec ently +ĠE ar +Ġc e +ĠSt ar +ĠC ong +rough t +Ġbl ood +Ġhop e +Ġcom ment +ain t +Ġar ri +il es +Ġpartic ip +ough t +ri ption +0 8 +4 9 +Ġg ave +Ġse lect +Ġkill ed +sy ch +Ġgo es +i j +Ġc oll +Ġimp act +at ives +ĠS er +0 9 +ĠAug ust +Ġb oy +d e +ĠD es +Ġf elt +U S +Ġexpect ed +Ġim age +ĠM ark +cc ording +o ice +E C +ĠM ag +en ed +h old +ĠP ost +Ġpre vent +N o +Ġinvol ved +Ġey es +Ġquick ly +A t +un k +Ġbeh av +Ġ ur +Ġl ed +c ome +e y +Ġcand id +Ġear lier +Ġfoc us +et y +P ro +led ge +ix ed +ill ed +Ġpop ular +A P +Ġset t +l ight +Ġvar ious +in ks +Ġlevel s +Ġro ad +ell ig +ab les +he l +itte e +ĠG ener +y pe +Ġhe ard +ic les +Ġm is +Ġus ers +ĠS an +Ġimpro ve +Ġf ather +Ġse arch +The y +v il +Ġprof ess +Ġkn ew +Ġl oss +Ġev ents +6 5 +Ġb illion +0 7 +0 2 +ĠNew s +ĠA M +Ġco ver +w here +ens ion +Ġb ott +Ġare as +en ces +op e +ĠTw itter +a el +Ġget s +ĠGo ogle +Ġs n +i ant +Ġv ote +Ġnear ly +Ġinclud ed +Ġrec ogn +z z +m m +al ed +Ġhappen ed +0 4 +Ġh ot +Ġwho se +Ġc ivil +Ġsu ff +o es +it iz +ĠSy ri +Ġresp ond +Ġh on +Ġfeat ures +Ġeconom ic +ĠApr il +r im +Ġtechn ology +Ġo ption +ag ing +Ġpur ch +R e +Ġl at +ch ie +is l +Ġrec omm +u f +Ġtr aining +Ġeffect s +Ġf ast +Ġ201 0 +Ġocc ur +Ġwebs ite +Ġem ail +Ġs ens +e ch +Ġo il +Ġinf lu +Ġcurrent ly +ĠS ch +ĠAd d +Ġgo al +Ġsc ient +Ġcon v +1 00 +em y +Ġdec ided +Ġtra vel +Ġm ention +L L +0 3 +Ġe lection +Ġph one +Ġlook s +Ġsit uation +Ġc y +Ġh or +b ed +ĠCour t +a ily +av es +Ġqu ality +ĠCom p +w ise +Ġt able +Ġst aff +ĠW ind +et t +Ġtri ed +ide red +Ġadd ition +Ġb ox +Ġl ack +ar ily +Ġw ide +Ġm id +Ġbo ard +ys is +Ġant i +h a +Ġd ig +en ing +Ġd ro +C on +6 8 +Ġsl ow +b ased +se qu +Ġp ath +E x +ak er +Ġwork ed +Ġp en +Ġeng ine +Ġlook ed +ĠSu per +ĠS erv +Ġvict im +U n +Ġproper ty +Ġint rodu +Ġexec ut +ĠP M +L e +Ġcol or +ĠM ore +Ġ6 0 +Ġnet work +Ġd ate +c ul +id ge +Ġext ra +3 1 +Ġs le +6 7 +Ġw ond +Ġreport s +j ust +ĠAust ral +Ġcap ital +Ġen s +Ġcomm and +Ġallow ed +Ġpre p +Ġca pt +h ib +Ġnum bers +ch an +Ġf air +m p +om s +Ġre ach +W ith +t ain +Ġbro ad +Ġcou ple +ec ause +ly ing +ĠF eb +Ġsc reen +Ġl ives +Ġpri or +ĠCong ress +A r +Ġappro ach +Ġe mer +ar ies +ĠD is +s erv +ĠN e +Ġbu ilt +c ies +Ġre pe +Ġrul es +for ce +ĠP al +Ġfin ancial +Ġcons idered +ĠCh ar +n ces +ĠI S +Ġb rought +Ġb i +i ers +ĠS im +O P +Ġproduct s +Ġvis it +Ġdoc ument +Ġcon duct +Ġcomplete ly +in ing +ĠCal if +ib ly +Ġwr itten +ĠT V +em ents +Ġd raw +O ne +Ġpub lished +Ġsec ret +r ain +he t +ĠF acebook +ond ay +ĠU p +Ġsex ual +Ġth ous +ĠP at +Ġ ess +Ġstand ard +Ġar m +g es +ect ion +Ġf ell +Ġfore ign +an i +ĠFr iday +Ġreg ular +in ary +Ġincre ased +Ġus ually +Ġdem on +Ġd ark +Ġadd itional +ro l +ĠO f +Ġprodu ction +! ! +und red +Ġintern ational +id ents +ĠF ree +rou p +Ġr ace +Ġm ach +Ġh uge +A ll +le ar +ove mber +Ġto wn +Ġatt ention +ĠO ff +y ond +ĠThe n +f ield +Ġter ror +ra z +ĠB o +Ġmeet ing +ĠP ark +Ġar rest +Ġf ear +Ġa w +ĠV al +or ing +' , +Ġext reme +ar r +Ġwork ers +A fter +Ġ3 1 +n et +am ent +Ġdirect ly +Ġpop ulation +ub e +ĠOct ober +ĠI N +ĠJan uary +5 9 +ĠDav id +Ġc ross +ce mber +ĠF irst +Ġmess age +ir it +Ġn ation +Ġp oll +is ions +Ġansw er +n y +is ode +Ġcar ry +ĠRuss ia +Ġhe ar +eng th +ro y +Ġn atural +in ally +Ġdo g +m itted +Ġtr ade +Ġsub st +Ġmult iple +ĠAf ric +Ġf ans +Ġs ort +Ġgl obal +ic ation +ĠW ed +ar a +Ġa chie +Ġlangu age +ve y +Ġt al +Ġnecess ary +Ġdet ails +Ġs en +ĠS und +ĠRe g +ĠR ec +0 6 +Ġs il +ress ive +Ġmed ical +un ch +orn ia +Ġu nd +f ort +oc ks +ĠM onday +ues day +c raft +7 7 +ur t +Ġ ver +ĠH ill +Ġrece ive +Ġmor ning +es tern +Ġb ank +Ġs at +ir th +ĠH igh +Ġdev ice +ĠTH E +ĠCent er +Ġsaf e +Ġp le +ĠCanad a +Ġsystem s +Ġass ist +Ġsur v +Ġb attle +ĠS oc +vert is +S he +Ġp aper +Ġgrow th +Ġc ast +S c +Ġpl ans +ll ed +Ġpart s +Ġw all +Ġmove ment +Ġpract ice +im ately +Ġdis play +Ġsomet imes +om p +ĠP aul +ĠY es +k ing +5 8 +o ly +Ġs on +Ġav oid +ok es +ĠJ ew +Ġto wards +as c +Ġ // +ĠK ore +Ġtalk ing +Ġcor rect +Ġsp ent +ic ks +i able +e ared +Ġter m +Ġwant s +om ing +Ġ ut +Ġdou b +Ġfor ces +Ġp lease +6 9 +ĠN ovember +at form +ond on +Ġon es +Ġimmedi ately +ĠRuss ian +ĠM et +Ġde g +Ġparent s +C H +ĠAmeric ans +al y +ĠM od +Ġsh own +Ġcond itions +Ġst uff +Ġre b +ĠY our +Ġinclud es +n own +ĠS am +Ġexper ien +m ission +ĠE ven +augh t +Ġannoun ced +ĠRepublic an +Ġdeter min +Ġdescrib ed +ĠCount y +( ) +Ġdo or +Ġchang ed +Ġne igh +ĠH ere +Ġcle an +Ġp an +ĠDe cember +ĠEurope an +ir ing +ap ter +Ġcl ub +ĠT uesday +Ġp aid +ĠN et +Ġattack s +Ġcharact ers +Ġal one +Ġdirect or +d om +Ġ3 5 +Ġl oad +Ġr out +ĠCalif ornia +Ġfin ally +Ġr ac +Ġcont r +Ġexact ly +res h +p ri +ĠIs lam +Ġn ature +Ġcare er +Ġlat est +Ġcon vers +ĠS l +p ose +ci ent +ĠIn c +iv ity +8 8 +ĠA tt +ĠM or +nes day +Ġwe ight +k en +Ġnot e +Ġteam s +Ġ \ +air s +ĠG reen +Ġh undred +on ent +Ġstre ng +Ġcons ist +ic ated +Ġreg ul +Ġl ic +ast ic +Ġt en +urs day +ellig ence +ous ly +ĠU K +B I +Ġcost s +Ġind epend +ĠA P +Ġnorm al +Ġh om +Ġob vious +Ġs we +Ġst ar +Ġread y +ac her +Ġimp lement +g est +Ġs ong +ĠG et +ĠL ab +Ġinterest ing +us ing +Ġg iving +ĠSund ay +Ġet c +Ġm iddle +Ġrem ember +r ight +os ition +ut ions +Ġm ax +4 6 +Ġyour self +Ġdem and +Ġtreat ment +Ġd anger +ĠC ons +Ġgu y +ĠBrit ish +Ġphys ical +Ġrel ated +Ġrem ain +Ġcould n +Ġref er +Ġc itiz +b ox +EN T +bo ard +Ġin n +I G +er o +ĠSt reet +osp ital +ren ch +cher s +Ġst ra +O L +ag er +ĠA N +Ġeas ily +I A +en ge +in y +Ġcl os +ock ed +Ġus es +ĠC oun +I m +u ild +? ? +m ore +Ġan g +Ġwr ite +ol ute +5 7 +Ġlead er +Ġread ing +< / +Ġaut om +est s +4 3 +Ġleg isl +ĠG old +Ġdesign ed +ĠS T +ĠLe g +a res +Ġbe aut +ĠT ex +Ġappear s +Ġstru gg +ĠR om +Ġ 00 +Ġcho ice +Ġparticular ly +ĠF rom +op er +ĠL ondon +ann ed +Ġallow s +ob ile +Ġdiffere nce +âĢ ¢ +ĠV iew +ĠWed nesday +Ġal though +Ġrel ative +Ġapplic ation +ate ver +Ġare n +Ġmy self +Ġim ag +Ġdis e +Ġsoc iety +Ġfre qu +ĠEng lish +Ġpo or +ĠD ay +Ġwrit ing +Ġse ven +Ġstart ing +Ġb ud +Ġpr int +ĠTr ans +uf act +ĠSt ud +n ew +Ġcr im +Ġg ives +Ġco ol +a e +i ance +ĠGener al +Ġthink ing +Ġsa ve +Ġlim ited +ĠPart y +Ġmean ing +p en +ow ers +ĠJ ack +E M +Ġn ice +ru pt +Ġg as +Ġe ight +Ġfe et +Ġeff ort +Ġ ign +ic it +B l +co in +Ġop in +Ġbr ain +Wh ile +he st +ĠTh ursday +Ġwould n +augh ter +Ġtou ch +le ments +Ġstud ies +Ġcent er +c ont +or ge +Ġcomput er +Ġinvestig ation +P l +or ks +Ġ200 8 +Ġincre asing +Ġst ore +Ġcom ments +Ġb al +m en +Ġdo ll +Ġl iber +Ġw ife +Ġlaw s +atur day +it ness +Ġmod ern +ĠS k +Ġadminist ration +Ġopportun ity +Ġs al +Ġpower ful +M y +Ġclaim s +ĠEar th +ord s +Ġt itle +Ġes c +n ame +N ot +om en +Ġbe yond +Ġc amer +Ġse ll +it ute +ear ch +Ġapp l +im ent +4 2 +ĠAr t +Ġun f +Ġviol ence +ur g +ĠE ast +Ġcomp ared +Ġopt ions +Ġthrough out +Ġv s +ig r +. [ +ac hes +7 8 +Ġfil es +F L +E L +ar ian +ĠJ ames +ĠA ir +an ch +Ġdet ail +Ġpie ce +P S +Ġn amed +Ġeduc ation +Ġdri ve +Ġitem s +Ġstud ent +ic ed +: : +ic o +Ġth row +Ġsc ene +Ġcomple x +Ġ200 9 +Ġpre c +ĠB re +7 9 +Ġcon cept +Ġstat us +am ing +Ġd ied +Ġknow ledge +Ġbegin ning +O D +ru ary +Ġcertain ly +Ġgu ys +Ġsl ight +in n +ound s +Ġf ine +Ġf at +ic ations +Ġper haps +ĠA nt +Ġinc ome +Ġhtt ps +Ġmajor ity +port s +st on +Ġgreat er +Ġfe ed +ent ially +Ġsaf ety +Ġun ique +and om +Ġg one +Ġshow ed +Ġhist or +Ġcoun ter +i us +id a +Ġlead ing +i pe +Ġs end +ĠDon ald +er ve +Ġdef ense +ines e +Ġy es +ĠF ire +ĠMus lim +ra q +Ġcontin ued +os h +Ġprov ides +Ġpr ison +ĠP re +Ġhapp y +Ġeconom y +Ġtr ust +ag s +ĠG ame +Ġweap ons +um an +ĠC le +it ation +Ġanal ysis +ĠT imes +Ġsc ience +- > +Ġfig ure +Ġdis app +ent y +Ġsoft ware +Ġu lt +Ġoffic ers +N ew +I s +Ġrem ains +ĠInd ia +Ġp sych +ri ef +Ġc at +es c +Ġob serv +Ġst age +ĠD ark +Ġent er +ch ange +Ġpass ed +Ġdes pite +ĠO ut +Ġmov ie +r s +Ġv oice +m ine +ĠPl ay +Ġto ward +ĠT er +Ġreg ion +Ġval ues +or ters +Ġm ount +Ġoffic er +ĠO ther +b an +Ġh ous +w ood +ro om +I V +ĠS un +se e +ĠO ver +ro g +9 0 +Ġl ay +ĠT ur +a wn +Ġpress ure +ĠS ub +Ġbook s +ed om +ĠS and +A A +ag o +Ġre asons +f ord +Ġactiv ity +U T +N ow +ĠSen ate +ce ll +n ight +Ġcall s +in ter +Ġlet ter +ĠR ob +ĠJ e +Ġcho ose +ĠL aw +G et +B e +Ġro b +Ġtyp es +Ġpl atform +Ġqu arter +R A +ĠT ime +Ġmay be +ĠC r +9 5 +p re +Ġmov ing +Ġl if +Ġgo ld +Ġs om +Ġpat ients +Ġtr uth +ĠK e +ur ance +ant ly +m ar +Ġchar ge +ĠG reat +Ġce le +---------------- ---------------- +Ġro ck +ro id +an cy +Ġcred it +a ud +B y +ĠE very +Ġmov ed +ing er +rib ution +Ġn ames +Ġstra ight +ĠHe alth +ĠW ell +Ġfe ature +Ġr ule +Ġsc he +in ated +ĠMich ael +ber g +4 1 +il ed +b and +Ġcl ick +ĠAng el +on ents +Â Ń +ĠI raq +ĠS aturday +Ġa ware +p art +Ġpat tern +O W +ĠL et +Ġgr ad +ign ed +Ġassoci ated +Ġst yle +n o +i ation +a ith +il ies +Ġst ories +ur ation +Ġindividual s +ĠâĢ ¦ +m iss +ĠAss oci +ish ing +ab y +Ġsum mer +ĠB en +Ġ3 2 +Ġar ch +ut y +ĠTex as +h ol +Ġfull y +Ġm ill +Ġfollow ed +ĠB ill +ĠInd ian +ĠSec ret +ĠB el +ĠFeb ruary +Ġjob s +Ġseem ed +ĠGo vern +i pped +Ġreal ity +Ġl ines +Ġp ark +Ġmeas ure +ĠO ur +I M +Ġbro ther +Ġgrow ing +Ġb an +Ġest im +Ġc ry +ĠS chool +Ġme chan +ĠO F +ĠWind ows +Ġr ates +ĠO h +Ġpos itive +Ġcult ure +ist ics +ic a +Ġh ar +y a +ite ly +i pp +Ġm ap +en cies +ĠWill iam +I I +ak ers +5 6 +ĠM art +ĠR em +Ġal tern +it ude +Ġco ach +row d +D on +Ġk ids +Ġj ournal +Ġcor por +Ġf alse +Ġwe b +Ġsle ep +Ġcont ain +Ġst o +Ġb ed +iver se +ĠR ich +ĠCh inese +Ġp un +Ġme ant +k nown +Ġnot ice +Ġfavor ite +a ven +Ġcond ition +Ġpur pose +) ) +Ġorgan ization +Ġchall eng +Ġman ufact +Ġsus p +ĠA c +Ġcrit ic +un es +uc lear +Ġm er +vent ion +Ġ8 0 +Ġm ist +ĠU s +ĠT or +htt p +ol f +Ġlarg er +Ġadv ant +Ġrese ar +Ġact ions +m l +Ġke pt +Ġa im +, ' +c ol +Ġbenef its +if ying +Ġact ual +ĠIntern ational +Ġveh icle +Ġch ief +Ġeff orts +ĠLe ague +ĠM ost +Ġwa it +Ġad ult +Ġover all +Ġspe ech +Ġhigh ly +Ġfem ale +Ġer ror +Ġeffect ive +5 4 +Ġenc our +w ell +Ġfail ed +Ġcons erv +Ġprogram s +Ġt rou +Ġa head +5 00 +vertis ement +I P +ĠF ound +p ir +Ġ % +Ġcr ime +and er +Ġloc ation +ĠI ran +Ġbehav ior +az ing +Ġr are +Ġem b +Ġca used +Ġsh ip +Ġact ive +Ġcont ribut +Ġg reen +Ġac qu +Ġref lect +ven ue +Ġf irm +Ġb irth +] . +Ġclear ly +Ġem ot +Ġag ency +ri age +Ġmem ory +9 8 +S A +ĠSe e +ac ing +C C +Ġbig gest +Ġr ap +Ġbas ic +Ġb and +e at +Ġsus pect +ĠM ac +Ġ9 0 +m ark +ist an +Ġsp read +am s +k i +as y +ra v +ĠR ober +Ġdemon str +r ated +Ġabs olute +Ġpl aces +Ġim pl +ibr ary +Ġc ards +Ġdest roy +Ġv irt +ve re +Ġapp eared +y an +p oint +Ġbe g +Ġtem per +s pe +ant ed +ear s +ĠD irect +Ġl ength +Ġbl og +am b +Ġint eg +Ġres ources +ac c +if ul +Ġsp ot +Ġfor ced +Ġthous ands +ĠMin ister +Ġqu al +ĠF rench +at ically +Ġgener ally +Ġdr ink +Ġth us +I L +od es +Ġappro pri +ĠRe ad +Ġwh om +Ġey e +Ġcol lege +Ġ4 5 +ire ction +Ġens ure +Ġapp arent +id ers +Ġrelig ious +Ġmin or +ol ic +Ġt ro +ĠWh y +rib ute +m et +Ġprim ary +Ġdevelop ed +Ġpe ace +Ġsk in +st e +av a +Ġbl ue +Ġfam ilies +Ġ ir +Ġapp ly +Ġin form +ĠSm ith +C T +i i +Ġlim it +Ġres ist +........ ........ +um n +Ġconf lic +Ġtw e +ud d +ĠT om +Ġl iter +qu e +b on +Ġha ir +Ġevent ually +Ġp us +Ġhelp ed +Ġag g +or ney +ĠApp le +Ġf it +ĠS ur +Ġpre m +Ġs ales +Ġsecond s +Ġstreng th +Ġfeel ing +¿ ½ +Ġt our +Ġknow s +o om +Ġex erc +Ġsom ew +ï ¿½ +> > +Ġsp okes +Ġide as +Ġreg ist +so ft +ĠD el +ĠP C +Ġpro pos +Ġlaun ch +Ġbott om +T H +ĠP lease +v est +it z +ĠIn ter +Ġsc ript +Ġr at +ar ning +Ġ il +ĠJ er +ĠA re +Ġwh atever +ok en +ci ence +Ġmod e +Ġag ree +Ġs ources +Ġinit ial +Ġrest rict +Ġwond er +us ion +## ## +ĠS il +vil le +Ġb urn +t w +as ion +Ġ £ +Ġn or +u ing +Ġre ached +Ġs un +Ġc ateg +ig ration +Ġc ook +Ġprom ot +Ġm ale +Ġcl imate +Ġf ix +Ġalleg ed +U R +all ed +Ġim ages +C ont +ot a +Ġschool s +i os +Ġd rop +Ġst ream +ĠM o +Ġprevious ly +al ing +Ġp et +Ġdou ble +Ġ( @ +ann el +Ġdef ault +t ies +Ġr ank +ĠD ec +ĠCoun cil +Ġweap on +Ġst ock +Ġanal y +ĠSt r +Ġpict ure +ĠPol ice +f erence +Ġcent ury +Ġcitiz ens +Ġon to +Ġexp and +Ġhe ro +ĠS ol +Ġw ild +Ġupd ate +Ġcustom ers +r ont +d ef +Ġl ik +Ġcrim inal +ĠChrist ian +S P +7 6 +Ġle aving +Ġother wise +ĠD ist +Ġbas is +5 2 +5 3 +ic ip +ĠB er +Ġrecomm end +Ġfl oor +Ġc rowd +ol es +Ġ7 0 +Ġcent ral +ĠE v +Ġd ream +Ġdown load +Ġconf ir +ĠTh om +Ġwind ow +Ġhapp ens +Ġun it +Ġt end +Ġs pl +Ġbec omes +Ġfight ing +Ġpred ict +ĠP ress +ĠP ower +Ġhe avy +ak ed +Ġf an +or ter +ate gy +B A +iz es +Ġsp end +H ere +Ġ200 7 +Ġad op +ĠH am +Ġfoot ball +ĠP ort +od ay +5 1 +amp ions +Ġtrans fer +h t +Ġ3 8 +ter m +ac ity +Ġb ur +] , +tern al +r ig +b ut +Ġthere fore +ĠB ecause +res p +re y +Ġm ission +S ome +Ġnot ed +Ġass um +Ġdise ase +Ġed it +Ġprog ress +r d +ĠB rown +oc al +Ġadd ing +Ġra ised +ĠAn y +Ġt ick +Ġsee ing +ĠPe ople +Ġagre ement +Ġser ver +Ġw at +Ġdeb ate +Ġsupp osed +il ing +Ġlarg est +Ġsuccess ful +ĠP ri +ĠDemocr atic +Ġj ump +ĠSyri a +Ġown ers +Ġoff ers +Ġshoot ing +Ġeff ic +se y +Ġha ven +ver se +te red +ĠL ight +im al +ĠB ig +Ġdef end +Ġbe at +Ġrecord s +% ) +Ġsc en +Ġemploy ees +Ġdev ices +he m +Ġcom mer +ĠM ex +Ġbenef it +ĠPro f +Ġil leg +Ġsur face +ĠAl so +Ġh arm +ing ly +w ide +ĠA lex +Ġsh ut +ĠC ur +Ġl ose +p m +Ġchall enge +se mb +Ġst ation +Ġint elligence +Ġacc ur +ĠFl or +Ġrequ ires +ĠM al +b um +Ġh ospital +Ġsp irit +Ġoff ered +Ġprodu ce +ĠComm un +Ġcreat ing +Ġcr is +s pect +Ġend ed +Ġd aily +Ġvot ers +land s +i as +i h +on a +Ġsm art +ĠOff ice +ĠL ord +ri al +ĠIntern et +Ġcirc um +Ġextreme ly +' . +Ġopin ion +ĠM il +Ġg ain +B S +ĠF in +y p +Ġuse ful +Ġbud get +Ġcom fort +is f +Ġback ground +el ine +Ġep isode +Ġen emy +Ġtri al +Ġestab lish +d ate +ĠC ap +Ġcontin ues +Ġshow ing +ĠUn ion +w ith +Ġpost ed +ĠSy stem +Ġe at +ri an +Ġr ise +ĠGerman y +il s +Ġsign ed +Ġv ill +Ġgr and +m or +ĠEng land +Ġproject s +um ber +Ġconf erence +z a +Ġrespons ible +ĠAr ab +Ġlearn ed +âĢĶ âĢĶ +i pping +ĠGe orge +O C +Ġreturn ed +ĠAustral ia +Ġb rief +Q u +Ġbr and +ill ing +ab led +Ġhig hest +Ġtr ain +ĠComm ission +wh ile +Ġn om +cept ion +Ġm ut +ĠBl ue +Ġinc ident +v ant +8 6 +ĠI D +Ġn uclear +7 4 +ĠL ike +ĠR E +ĠM icro +l i +m ail +Ġcharg es +8 9 +Ġad just +ad o +Ġear th +N A +Ġpr ices +P A +Ġd raft +Ġrun s +Ġcandid ate +ens es +Ġmanag ement +ĠPh il +ĠM iss +Ġte ach +g ram +Ġunderstand ing +a it +ic ago +A dd +ĠE p +sec ut +Ġsepar ate +Ġinst ance +Ġe th +Ġun less +**** **** +ĠF ore +in ate +Ġoper ations +S p +Ġf aith +g ar +ĠCh urch +ron ic +Ġconf ig +os ure +Ġactiv ities +Ġtrad itional +Ġ3 6 +Ġd irection +Ġmach ine +Ġsur round +Ġp ush +un ction +ĠE U +Ġeas ier +Ġarg ument +G B +Ġm icro +Ġsp ending +iz ations +Ġthe ory +ad ow +Ġcall ing +ĠL ast +Ġd er +Ġinflu ence +Ġcomm it +Ġph oto +Ġun c +ist ry +g n +ast e +ack s +Ġdis p +ad y +d o +ĠG ood +Ġ ` +Ġw ish +Ġreve aled +Âł Âł +l ig +Ġen force +ĠComm ittee +Ġche m +Ġmil es +Ġinterest ed +Ġsol ution +ic y +in ct +Ġ- > +ĠD et +Ġrem oved +Ġcomp ar +e ah +Ġpl ant +ĠS ince +Ġachie ve +Ġadvant age +Ġslight ly +b ing +Ġpl aced +u nder +201 5 +ĠM ad +Ġt im +os es +Ġc ru +ĠR ock +Ġmost ly +Ġneg ative +Ġset ting +Ġprodu ced +Ġm ur +Ġconnect ion +ĠM er +Ġdri ver +Ġexecut ive +Ġass ault +Ġb orn +ĠV er +t ained +Ġstruct ure +Ġredu ce +Ġdec ades +Ġd ed +u ke +ĠM any +idd en +Ġle ague +S e +Ġjo in +Ġdis co +Ġd ie +c ks +act ions +Ġass ess +ag n +Ġgo als +our s +I R +Ġsen ior +ill er +m od +ip ment +oc ol +u y +ĠQ ue +Ġpart ies +ir gin +Ġle arning +it able +Ġstre et +Ġcamer a +A pp +Ġsk ills +b re +c ious +Ġcele br +ĠFr anc +Ġexist ing +Ġwill ing +l or +Ġ id +ĠSp ace +Ġcrit ical +ĠL a +ortun ately +Ġser ve +Ġc old +Ġspec ies +T S +Ġanim als +ĠB ay +Ġold er +ĠU nder +est ic +ĠT re +Ġte acher +Ġpre fer +v is +Ġth read +ĠM att +Ġmanag er +ãĥ » +Ġprofess ional +ĠV ol +Ġnot es +The se +ul a +Ġf resh +ent ed +u zz +ed y +clus ion +ĠR el +Ġdoub t +E O +Ġopen ed +ĠB it +Ad vertisement +Ġgu ess +ĠU N +Ġse qu +Ġexpl ain +ott en +Ġatt ract +ak s +Ġstr ing +Ġcont ext +oss ible +ĠRepublic ans +Ġsol id +Ġc ities +Ġask ing +Ġr andom +u ps +ur ies +ar ant +dd en +g l +ĠFlor ida +Ġdep end +ĠSc ott +Ġ3 3 +Ġi T +ic on +Ġmention ed +Ġ2 000 +Ġclaim ed +Ġdefin itely +ul f +Ġc ore +Ġopen ing +ĠCon st +wh ich +ĠT ra +A G +7 2 +Ġbelie ved +ad a +Ġ4 8 +ĠSec urity +yr ight +ĠP et +ĠL ou +Ġhold ing +======== ======== +Ġ ice +Ġb row +Ġauthor ities +h ost +w ord +Ġsc ore +ĠD iv +Ġcell s +Ġtrans l +Ġneigh bor +Ġrem ove +u ct +Ġdist rict +ĠA ccording +Ġwor se +Ġconcern s +Ġpresident ial +Ġpolic ies +ĠH all +7 3 +Ġh us +A Y +Ġ200 6 +ĠJ ud +Ġindepend ent +ĠJust ice +ili ar +pr int +igh ter +Ġprotect ion +z en +Ġsu dden +h ouse +ĠJ es +P R +ĠIn f +Ġb ul +Ġ _ +ĠServ ice +ĠP R +Ġstr ategy +ff ect +Ġgirl s +Ġmiss ing +oy al +ĠTe am +ul ated +Ġd at +Ġpolit ics +ab or +A ccording +Ġspe ll +Ġg raph +ort hern +T C +A b +Ġlab or +is her +Ġk ick +ĠiT unes +Ġstep s +pos es +Ġsmall er +E n +ber t +Ġro ll +Ġresear chers +Ġcl osed +Ġtrans port +Ġlaw y +________ ________ +ĠCh icago +Ġas pect +Ġn one +Ġmar riage +9 6 +Ġe lements +ĠF re +ĠS al +Ġd ram +F C +t op +e qu +Ġhe aring +Ġsupport ed +Ġtest ing +co hol +Ġmass ive +Ġst ick +Ġgu ard +is co +ph one +F rom +How ever +Ġb order +Ġcop y +ograph y +l ist +7 1 +Ġown er +cl ass +ru it +r ate +ĠO nce +Ġdig ital +Ġt ask +ER S +Ġinc red +t es ++ + +ĠFr ance +Ġb reat +ow l +Ġiss ued +ĠW estern +Ġdet ect +Ġpart ners +Ġsh ared +ĠC all +Ġcan cer +ac he +rib e +Ġexpl ained +Ġhe at +{ " +Ġinvest ment +ĠB ook +Ġw ood +Ġtool s +ĠAl though +Ġbelie f +Ġcris is +Ġg e +ĠM P +Ġoper ation +ty pe +~ ~ +g a +Ġcont ains +ant a +Ġexp ress +ĠG roup +ĠJ ournal +k a +Ġam b +ĠUS A +Ġfind ing +Ġfund ing +h ow +Ġestab lished +ide os +Ġdeg ree +Ġdanger ous +ang ing +Ġfre edom +pp ort +out hern +Ġch urch +Ġc atch +ĠTw o +Ġpres ence +ĠGu ard +U p +Ġauthor ity +ĠPro ject +Ġbut ton +Ġcon sequ +Ġval id +Ġwe ak +Ġstart s +Ġref erence +ĠM em +" ) +U N +or age +ĠO pen +Ġcol lection +y m +g ency +Ġbeaut iful +ro s +Ġtell s +Ġwa iting +n el +Ġprov iding +ĠDemocr ats +Ġd aughter +Ġm aster +Ġpur poses +ĠJapan ese +Ġequ al +Ġturn s +Ġdoc uments +Ġwatch ing +R es +Ġr an +201 4 +Ġre ject +ĠKore a +Ġvictim s +Le vel +ere nces +Ġw itness +Ġ3 4 +Ġre form +com ing +Ġocc up +Ġc aught +Ġtra ffic +ad ing +Ġmod els +ar io +Ġserv ed +Ġb atter +u ate +ĠSecret ary +Ġagre ed +Ġtr uly +yn am +ĠR et +Ġun its +ĠRes earch +h and +az ine +ĠM ike +Ġvar iety +ot al +Ġam azing +Ġconfir med +Ġentire ly +Ġpurch ase +Ġe lement +Ġc ash +Ġdeter mine +D e +Ġc ars +ĠW all +â ĸ +Ġview s +Ġdrug s +Ġdep artment +ĠSt ep +u it +Ġ3 9 +as ure +ĠCl ass +Ġc overed +ĠB ank +Ġme re +u ana +Ġmult i +Ġm ix +Ġun like +lev ision +Ġsto pped +Ġs em +ĠG al +ul es +Ġwe l +ĠJohn son +l a +Ġsk ill +Ġbec oming +ri e +Ġappropri ate +f e +ell ow +ĠPro t +ul ate +oc ation +Ġweek end +od ies +Ġsit es +Ġanim al +ĠT im +Ġsc ale +Ġcharg ed +Ġinst ruct +ill a +Ġmethod s +Ġc ert +Ġjud ge +ĠH el +Ġdoll ars +Ġstand ing +ĠS qu +Ġdeb t +l iam +Ġdri ving +ĠS um +ĠEd ition +Ġal bum +and on +I F +ĠU k +6 3 +ad er +Ġcommer cial +es h +ĠGovern ment +Ġdisc overed +Ġout put +ĠHill ary +ĠCar ol +Ġ200 5 +Ġab use +anc ing +Ġsw itch +Ġann ual +T w +Ġst ated +ag ement +in ner +Ġdem ocr +Ġres idents +Ġallow ing +Ġfact ors +od d +Ġf uck +em ies +Ġoccur red +ot i +Ġn orth +ĠP ublic +Ġinj ury +Ġins urance +C L +oll y +ã Ģ +Ġrepe ated +Ġar ms +ang ed +Ġconst ruction +Ġf le +P U +ic ians +Ġfor ms +ĠMc C +ant ic +Ġm ental +p ire +Ġequ ipment +Ġf ant +Ġdiscuss ion +Ġregard ing +k in +ar p +Ġch air +og ue +Ġpro ceed +ĠI d +O ur +Ġmur der +M an +Ġ4 9 +as p +Ġsupp ly +Ġin put +Ġwe alth +liam ent +Ġpro ced +or ial +ĠSt at +ĠN FL +hen s +ĠInst itute +Ġput ting +ourn ament +et ic +Ġloc ated +Ġk id +er ia +r un +Ġpr inc +Ġ ! +go ing +ĠB et +Ġcl ot +Ġtell ing +Ġprop osed +i ot +or ry +Ġfund s +g ment +ĠL ife +Ġb aby +ĠB ack +Ġsp oke +Im age +Ġear n +ĠA T +g u +Ġex change +ĠL in +ov ing +Ġp air +M ore +az on +Ġarrest ed +Ġkill ing +c an +ĠC ard +y d +Ġident ified +Ġm obile +Ġthan ks +ony m +ĠF orm +Ġhundred s +ĠCh ris +ĠC at +Ġtre nd +h at +ĠA v +om an +Ġelect ric +ĠW il +S E +O f +Ġrest aur +ot ed +Ġtr ig +Ġn ine +Ġb omb +Wh y + ¯ +Ġco verage +Ġapp eal +ĠRober t +ĠS up +Ġfin ished +Ġfl ow +Ġdel iver +Ġcal cul +Ġphot os +Ġph il +Ġpie ces +Ġapp re +k es +Ġr ough +D o +Ġpart ner +Ġconcern ed +Ġ3 7 +ĠG en +C ol +ct ors +Ġ= > +st ate +Ġsuggest ed +ĠFor ce +C E +Ġher self +ĠPl an +w orks +o oth +ren cy +Ġcor ner +Ġhus band +Ġintern et +ĠA ut +em s +os en +ĠAt l +g en +Ġbal ance +6 2 +Ġsound s +te xt +Ġar r +ov es +Ġmill ions +Ġrad io +Ġsat isf +ĠD am +M r +G o +S pe +Ġcomb at +r ant +ĠG ree +Ġf uel +Ġdist ance +Ġtest s +Ġdec re +ĠE r +Ġman aged +D S +Ġt it +Ġmeas ures +ĠL iber +Ġatt end +as hed +ĠJ ose +ĠN ight +d it +ĠN ov +ĠE nd +out s +Ġgener ation +Ġadv oc +y th +Ġconvers ation +ĠS ky +act ive +ce l +ri er +ĠFr ank +Ġg ender +Ġcon cent +Ġcar ried +and a +ĠV irgin +Ġarri ved +ic ide +ad ed +Ġfail ure +Ġmin imum +le ts +Ġwor st +Ġkeep ing +Ġint ended +Ġilleg al +Ġsub sc +Ġdetermin ed +Ġtri p +Y es +Ġra ise +Ġ ~ +Ġfeel s +Ġpack age +ĠJ o +h i +201 6 +re al +Ġf ra +Ġsy mb +M e +uck y +p ret +ĠK h +ĠEd it +ĠWe b +em ic +ĠCol or +Ġjust ice +I nt +Ġfar m +ck now +" > +el ess +Ġredu ced +Ġ5 00 +x x +ĠR ad +ĠW ood +Ġcl in +Ġhy p +il er +ur a +k ins +8 5 +6 1 +ĠThe ir +ĠM ary +Ġs an +Ġno vel +ĠWh o +Ġcap acity +Ġimp ossible +Ġpl ays +Ġmin ister +ij uana +ic ate +ĠS et +Ġf ram +Ġ ing +Ġcommun ities +ĠF BI +it a +Ġb on +Ġstr ateg +Ġinterest s +l ock +g ers +m as +ĠAN D +Ġconflic t +Ġrequire ments +Ġs ac +Ġoper ating +in i +rel ated +Ġcomm itted +Ġrelative ly +Ġs outh +¯ ¯ +Ġaff ord +Ġident ity +Ġdec isions +Ġacc used +pl ace +Ġvict ory +o ch +i at +N ame +C om +t ion +ed s +Ġsee k +Ġt ight +ĠIm ages +Ġinit i +Ġhum ans +Ġfam iliar +Ġaud ience +Ġintern al +vent ure +Ġs ides +ĠT O +Ġd im +Ġcon clud +Ġapp oint +Ġenforce ment +ĠJ im +ĠAssoci ation +Ġcircum st +ĠCanad ian +Ġjo ined +Ġdiffere nces +ĠL os +Ġprot est +Ġtw ice +w in +Ġgl ass +ars h +ĠAr my +Ġexp ression +Ġdec ide +Ġplan ning +an ia +Ġhand le +ĠMicro soft +ĠN or +Ġmax imum +ĠRe v +Ġse a +Ġev al +Ġhel ps +re f +Ġb ound +Ġm outh +Ġstand ards +Ġcl im +ĠC amp +ĠF ox +cl es +Ġar my +ĠTe chn +ack ing +x y +S S +Ġ4 2 +Ġbu g +ĠUk rain +ĠM ax +ĠJ ones +ĠSh ow +l o +Ġplan et +Ġ7 5 +Ġwin ning +Ġf aster +Ġspe ct +Ġbro ken +T R +Ġdef ined +Ġhealth y +Ġcompet ition +htt ps +ĠIs land +ĠF e +Ġannoun ce +ĠC up +ĠInst ead +Ġcl ient +Ġposs ibly +se ction +ock et +l ook +Ġfin ish +Ġcre w +Ġres erv +Ġed itor +Ġh ate +Ġs ale +Ġcontro vers +Ġp ages +w ing +Ġnum er +Ġopp osition +Ġ200 4 +Ġref uge +Ġfl ight +Ġap art +ĠL at +A meric +ĠAfric a +Ġapplic ations +ĠPal est +ĠB ur +Ġg ar +ĠSoc ial +Ġup gr +Ġsh ape +Ġspe aking +ans ion +a o +ĠS n +Ġwor ry +ĠBrit ain +P lease +rou d +Ġh un +Ġintrodu ced +Ġd iet +I nd +ĠSec ond +Ġfun ctions +ut s +ĠE ach +ĠJe ff +Ġst ress +Ġaccount s +Ġgu arant +ĠAn n +ed ia +Ġhon est +Ġt ree +ĠAfric an +ĠB ush +} , +Ġs ch +ĠOn ly +Ġf if +ig an +Ġexerc ise +ĠEx p +Ġscient ists +Ġlegisl ation +ĠW ork +ĠS pr +à Ĥ +ĠH uman +Ġ è +Ġsur vey +Ġr ich +ri p +Ġmain tain +Ġfl o +Ġleaders hip +st ream +ĠIslam ic +Ġ 01 +ĠCol lege +Ġmag ic +ĠPr ime +Ġfig ures +201 7 +ind er +x ual +ĠDe ad +Ġabsolute ly +Ġfour th +Ġpresent ed +resp ond +rib le +Ġal cohol +at o +ĠD E +por ary +Ġgr ab +Ġvar i +Ġqu ant +ĠPh oto +Ġpl us +r ick +ar ks +Ġaltern ative +Ġp il +Ġappro x +th at +Ġobject s +ĠR o +ĠAnd roid +Ġsignificant ly +ĠR oad +k ay +R ead +av or +Ġa cknow +ĠH D +ĠS ing +O r +ĠM ont +Ġun s +pro f +Ġneg oti +ĠAr ch +ik i +Ġte levision +ĠJew ish +Ġcomm ittee +Ġmot or +Ġappear ance +Ġs itting +Ġstri ke +ĠD own +com p +ĠH ist +Ġf old +ac ement +ĠLou is +Ġbel ong +ĠâĢ ¢ +Ġm ort +Ġprep ared +Ġ6 4 +ĠM aster +Ġind eed +ĠD en +Ġre nt +T A +our ney +ar c +S u +9 7 +Ġadv ice +Ġchang ing +Ġlist ed +Ġlaun ched +is ation +ĠP eter +is hes +Ġl ived +ĠM el +ĠSup reme +ĠF ederal +Ġ) ; +ruct ure +Ġset s +Ġphil os +u ous +Ġ ł +Ġappl ied +ĠN OT +Ġhous ing +ĠM ount +Ġo dd +Ġsu st +D A +ffic ient +Ġ ? +ol ved +Ġp owers +Ġth r +Ġrem aining +ĠW ater +L C +Ġca uses +ãģ ® +Ġman ner +ad s +Ġsuggest s +Ġend s +stand ing +f ig +ĠD un +id th +Ġg ay +Ġter min +ĠAngel es +M S +Ġscient ific +Ġco al +ap ers +b ar +ĠThom as +Ġsy m +ĠR un +th is +P C +igr ants +Ġmin ute +ĠDist rict +cell ent +Ġle aves +Ġcomple ted +am in +Ġfoc used +Ġmon itor +Ġveh icles +M A +ĠM ass +ĠGr and +Ġaffect ed +itution al +Ġconst ruct +Ġfollow s +Ġt on +re ens +Ġh omes +ĠE xt +ĠLe vel +r ast +ĠI r +Ġel im +Ġlarge ly +ĠJ oe +Ġvot es +all s +Ġbusiness es +ĠFound ation +ĠCent ral +Ġy ards +Ġmaterial s +ul ner +Ġgu ide +Ġclos er +um s +Ġsp orts +ed er +J ust +Ġtax es +8 4 +ĠO ld +Ġdec ade +ol a +Ġv ir +Ġdro pped +Ġdel ay +it ect +Ġsec ure +ste in +le vel +Ġtre ated +Ġfil ed +ain e +Ġv an +Ġm ir +Ġcol umn +ict ed +e per +Ġro t +Ġcons ult +Ġent ry +Ġmar ijuana +ĠD ou +Ġapparent ly +ok ing +clus ive +Ġincre ases +an o +Ġspecific ally +Ġte le +ens ions +Ġrelig ion +ab ilities +Ġfr ame +ĠN ote +ĠLe e +Ġhelp ing +Ġed ge +ost on +Ġorgan izations +à ĥ +ĠB oth +hip s +Ġbig ger +Ġbo ost +ĠSt and +Ġro w +ul s +ab ase +Ġr id +L et +are n +ra ve +Ġst ret +P D +Ġv ision +Ġwe aring +Ġappre ci +Ġa ward +ĠU se +Ġfact or +w ar +ul ations +) ( +Ġg od +Ġter rit +Ġpar am +ast s +8 7 +Ġen emies +ĠG ames +F F +Ġacc ident +W ell +ĠMart in +T ER +Ġat h +ĠHe ll +Ġfor g +Ġve ter +ĠMed ic +f ree +Ġst ars +Ġexp ensive +Ġac ad +ra wn +ĠW he +Ġl ock +Ġform at +Ġsold iers +s m +Ġag ent +Ġrespons ibility +or a +ĠS cience +Ġrap id +Ġt ough +ĠJes us +Ġbelie ves +M L +Ġwe ar +le te +Ãĥ ÃĤ +ĠD ri +Ġcomm ission +ĠB ob +O h +ap ed +Ġwar m +ÃĥÃĤ ÃĥÃĤ +Ġ200 3 +ort ion +Ġhas n +ust er +Ġun ivers +ĠI ll +Ġk ing +olog ies +9 4 +ĠT em +ĠM os +Ġpat ient +ĠMex ico +ce an +ĠDe ath +ĠSand ers +y ou +ĠC ast +ĠComp any +pt y +Ġhappen ing +F P +ĠB attle +Ġb ought +A m +M od +U s +ut ers +ĠC re +ĠTh ose +Ġ4 4 +is er +Ġs oul +ĠT op +ĠHar ry +ĠA w +Ġse at +ff ee +Ġrev olution +Ġ( " +ĠD uring +et te +Ġr ing +Ġoff ensive +Ġreturn s +Ġv ideos +Ġdis cl +Ġfam ous +en ced +ĠS ign +ĠR iver +Ġ3 00 +P M +ĠB us +ĠC H +Ġcandid ates +ard en +Ġpercent age +Ġvis ual +Ġthan k +Ġtrou ble +ner gy +Ġ200 1 +Ġpro ve +ash ion +Ġen h +ĠL ong +U M +Ġconnect ed +Ġposs ibility +O ver +Ġexper t +Ġl ibrary +art s +ĠDirect or +Ġfell ow +9 2 +ir ty +Ġd ry +Ġsign s +ĠL ove +Ġqu iet +f oot +Ġp ure +ĠH un +Ġf illed +ph as +ĠE lect +end ment +ĠEx pl +Ġun able +n s +m o +Ġv ast +ob e +Ġident ify +app ing +ĠCarol ina +g ress +Ġpro te +Ġf ish +Ġcircumst ances +raz y +ĠPh ot +Ġb odies +ĠM ur +Ġdevelop ing +ĠA R +Ġexperien ced +Ġsubst ant +ĠBo ard +es ome +Ġdom estic +Ġcomb ined +ĠP ut +Ġchem ical +ĠCh ild +Ġpo ol +ĠC y +Ġe gg +c ons +st ers +Ġh urt +Ġmark ets +Ġconserv ative +Ġsupp orters +Ġag encies +id el +O b +ur b +Ġ4 3 +ĠDef ense +y e +ĠA p +du le +Ġtemper ature +Ġconduct ed +ĠCh ief +Ġpull ed +Ġf ol +L ast +ont o +os is +V ER +D es +ĠP an +F irst +Ġadv ance +Ġlic ense +r ors +ĠJ on +Ġimag ine +Ġhe ll +Ġf ixed +Ġinc or +os ite +ĠL og +ick en +] : +Ġsurpr ise +h ab +Ġc raft +ol t +ĠJ ul +Ġd ial +Ġrele vant +Ġent ered +Ġlead s +ĠA D +ĠCle an +Ġpict ures +ess or +Ġal t +Ġpay ing +P er +ĠMark et +Ġupd ates +am ily +ĠT ype +ĠH ome +Ġ5 5 +semb ly +rom e +8 3 +Ġgreat est +Ġhe ight +Ġhe av +ain ts +Ġlist en +as er +ĠS H +Ġcap able +ac le +Ġpers pect +in ating +Ġoff ering +ry pt +ĠDe velop +ab in +r c +Ġbr ight +al ty +ar row +Ġsupp l +ind ing +ack ed +gy pt +ĠAn other +p g +ĠVirgin ia +ĠL u +Ġpl anned +Ġp it +Ġswe et +T ype +ĠD i +Ġtyp ically +ĠFranc isco +Ġpro spect +ĠD an +Ġte en +re es +Ġsc hed +Ġh ol +Ġsc r +Ġlot s +l ife +Ġnews p +Ġfor get +ĠN one +ĠM iddle +ĠR yan +ed d +Ġse vere +Ġsu it +ll er +9 3 +Ġcor respond +Ġexpl os +u ations +Ġfl ag +g ame +r id +Ġpr in +ĠD ata +Ġde ploy +ĠEn ter +su it +gh an +ĠM en +Ġthough ts +Ġmat ters +Ġad apt +ĠA ri +Ġf ill +Ġfor th +Ġs am +Ġ4 1 +Ġpay ment +ĠH or +Ġsp ring +du c +Ġl osing +Ġbring ing +F O +al a +Ġdist ribution +he red +b our +ĠIsrael i +om a +Ġcomb ination +Ġpl enty +V E +C an +ĠH aw +Ġper man +ĠSpe cial +Ġto w +Ġsee king +Ġexam ples +Ġclass es +c r +Ġbe er +Ġmov es +ĠI P +ĠK n +Ġpan el +E ven +Ġproper ly +Ġr is +Ġpl ug +Ġestim ated +E very +Ġdef ensive +ag raph +Ġpre gn +Ġinst it +ĠV ict +Ġvol ume +Ġpos itions +Ġl inks +ĠPro gram +ĠWe ek +ag ues +Ġtrans form +k er +ĠC EO +Ġc as +Ġopp onent +Ġtwe et +ĠC ode +Ġsh op +Ġf ly +Ġtal ks +Ġb ag +Ph one +Ġa id +Ġpl ants +Ġ6 5 +Ġatt orney +ar ters +qu est +ĠMag ic +Ġbeg ins +Ġmy ster +Ġenvironment al +Ġst orage +N N +Ġm arg +Ġs ke +Ġmet al +ell y +Ġord ered +Ġrem ained +Ġl oved +Ġprom pt +Ġupd ated +Ġexper ts +Ġwalk ing +Ġan cient +Ġperform ed +AT E +Ġne ither +i ency +Ġmanufact ure +ĠP ak +Ġselect ed +Ġm ine +Ġult imately +Ġexpl an +Ġlab el +ĠServ ices +ribut ed +Tr ump +Ġsy n +ĠU lt +S C +Ġme at +Ġg iant +ĠW ars +ĠO N +Ġad m +Ġinter pret +Ġeven ing +Ġev il +ĠB oston +ĠW ild +Ġ à +ĠBit coin +ĠAm azon +D r +ĠIn formation +Ġobvious ly +Ġadv anced +Ph oto +ol ar +Ġwe ather +Ġsymb ol +Ġso le +Ġpot entially +ost er +Ġorig inally +m un +3 00 +az e +ess ions +Ġde ck +Ġst ood +Ġyou th +ĠB ern +R ep +ĠT est +Ġbas ically +ot ic +Ġinvol ve +ol it +ly n +S ee +Ġair craft +Ġconf irm +E W +Ġmess ages +ĠRich ard +Ġk it +Ġpro hib +Ġv ulner +is ters +Ġexist ence +Ġturn ing +ĠS P +Ġdes ire +Ġfl at +Ġm ent +se ason +ang es +Ġneighbor hood +ĠL ake +AT ION +Ġpoint ed +b ur +Ġinn ov +uc ks +U L +Ġprofess or +Ġexp ressed +A B +ic ious +Ġ200 2 +ĠDe v +Ġs ession +Ġb are +s en +Ġdis s +ĠC ath +ĠP ass +ĠP oint +Ġdo ctor +or row +ail ed +ĠR ub +ĠD C +ĠChar l +p erson +Ġwrit er +igh ters +ure au +Ġob lig +Ġrecord ed +Ġbro ke +Ġord ers +il ty +Ġmot ion +in ity +l aw +ad ium +Ġimm igration +Ġcontr ast +Ġb att +Ġex cellent +Ġtechn ical +am i +Ġt un +Ġcl oud +ĠY ear +ge on +Ġcre ation +Ġstr ange +Ġa uth +Ġfor t +b orn +Ġext ent +ĠT oday +ĠCl ub +Ġr ain +Ġs ample +Ġaccept ed +Ġt act +Ġf ired +ĠS on +Ġstand s +Ġb oot +Ġ4 7 +Ġstat ements +Ġvers ions +Ġse lling +ound ed +Ġ199 0 +Ġwere n +ĠW atch +Ġexper iment +P ost +Ġret ail +ul ed +In st +un te +ãĥ ¼ +Ġdep art +Ġb ond +i very +om pl +Ġre action +ĠSyri an +ĠP ac +app ed +ani el +D P +Ġres olution +Ġre act +Ġappro ved +on om +m ond +ĠO ffic +-- - +Ġrepl ace +Ġt ack +Ġsp ort +Ġch ain +Ġemer gency +r ad +ĠPalest in +Ġ4 6 +Ġautom atically +Ġrout e +Ġp al +Ġb anks +ĠPar is +ĠMed ia +ro ad +ic ing +i xt +ist ed +Ġg rew +Ġco ord +ĠW here +om in +Ġsub s +� � +Ġ ± +Ġcorpor ate +Ġse lection +n oon +ĠRep ort +c s +clud ing +ord ers +anc he +ĠIt s +Ġslow ly +ĠE gypt +ĠA cc +Ġcol le +iqu es +E X +Ġattempt s +ur l +ĠC ross +Ġfind ings +ĠS C +ĠO R +Ġind ex +ens ity +ĠW ay +ĠL and +Ġsh ock +d is +Ġd ynam +Ġc art +m osp +S ince +i est +ĠB oy +Ġst orm +ĠCont in +201 3 +he w +il it +Ġess ential +iqu id +O ther +ive red +Ġreason able +A ct +Ġsub sequ +ĠP ack +ĠF ort +Ġconsider ing +Ġun iversity +l og +Ġmar ried +Ġill ust +ĠTr ue +£ ı +Ġnumer ous +rast ructure +Ġserious ly +Ġrefer red +u a +Ġconsist ent +on na +ĠRe al +ru ption +ci ples +Ġfact s +9 1 +ot es +er g +The n +Ġacc ompl +N ote +Ġre venue +Ġpass ing +Ġm al +e en +ĠY et +Ġg ather +ter day +ew ork +ĠA uthor +P e +Ġopt im +Ġr ub +Ġè £ı +Ġun known +st one +Ġun ion +ol ve +Ġopportun ities +Ġbrow ser +ĠW al +ĠC ost +Ġreport ing +st s +p et +Ġs and +Ġsudden ly +Ġsurpr ising +ĠV R +Ġsomew hat +ĠB as +ult ure +iz z +ĠC D +Ġchalleng es +Ġsett ings +Ġexperien ces +ĠF ull +Ġcan n +Ġrece iving +ES T +Ġj oint +Ġcult ural +Ġa st +8 2 +as tern +ce ived +ĠC ru +Ġb ull +p ired +am m +Ġfac ing +p ower +Ġb oss +ĠH ol +Ġinst r +Ġincreasing ly +Ġsh ift +Ġstre ets +ĠWilliam s +ab b +Ġl ie +Ġl augh +ĠC a +P L +Ġadult s +Ġcustom er +Ġob tained +Ġsupport ing +ht ml +f ire +Ġdetail ed +Ġpick ed +ĠR ight +ld er +E E +st ood +ĠK im +Ġw ire +Ġs ight +Ġdevelop ers +Ġpers ons +Ġs ad +Ġc up +Ġwar ning +Ġboy s +l ong +Ġb ird +f o +Ġw al +Ġobserv ed +Ġz one +iven ess +Ġch annel +c ript +Ġref used +ĠAg ain +Ġsu c +Ġspokes man +ĠRe f +r ite +ou ston +ãĥ ³ +ĠS her +Ġact s +ĠN ame +Ġstrugg le +ar ry +omet imes +Ġdisc rim +H T +Ġcateg ory +Ġreal ize +Ġemploy ee +ĠAf ghan +en ger +Ġgun s +ĠSte ve +ĠM ot +ĠO l +ok ed +Ġth ick +Ġfair ly +ill y +Ġsur ve +ĠM at +we ight +â Ķ +Ġtro ops +Ġag ents +Ġbatter y +Ġmot iv +à ¡ +S ec +d en +o very +L S +Ġfl u +Ġconf ident +ĠO per +Ġem pty +Ġp hen +Ġse ctor +Ġexc ited +Ġrem ote +ap h +o en +Ġdestroy ed +Ġmor al +ĠH P +ĠR on +Ġd ress +ĠB at +Ġl it +ĠM S +Ġa f +H L +r um +is ms +Ġshould n +Ġsym pt +ĠTor onto +het ic +Ġcar bon +Ġinstall ed +Ġviol ent +Ġsol ar +j a +Ġpract ices +Ġr ide +ĠP enn +Ġimpro ved +Ġaud io +Ġbehav i +ĠP S +Ġe ating +D ata +ĠRe view +p ass +cl aim +u ated +ang ers +c hen +Ġproper ties +Ġany where +An other +Ġbl ow +ĠJack son +Ġp roud +Ġplan e +l ines +Ġsqu are +Ġpro of +ans as +Ġtalk ed +m akers +Ġs ister +Ġhold s +Ġres ident +Ġ= = +Ġresist ance +Ġspl it +Ġpro secut +Ġconf idence +res ents +Ġcut s +Ġexcept ion +Ġz ero +Get ty +Ġcop yright +Ġtot ally +orm al +ific ations +ĠAustral ian +Ġs ick +Ġ1 50 +Ġhouse hold +Ġfe es +Ġdri vers +og en +ĠN Y +Ġnecess arily +Ġregul ations +ear ing +s l +Ġperspect ive +c are +ic ial +H is +Ġesc ape +Ġsurpr ised +ĠV an +ur rent +Ġv ac +8 1 +ĠTh us +Ġem phas +ĠCh ampions +ĠI ce +Ġn arr +Ġhead s +Ġca using +b el +f ortunately +ĠM a +Ġtarg ets +ci pl +Ġafter noon +Ġadd s +ĠMay be +ĠF our +ess ed +ple te +Ġus ual +ch o +ing u +Ġwith d +ĠE nergy +ĠE conom +O O +Ġart icles +Ġinj ured +Ġman age +Ġexpl ains +Ġdi agn +R ec +at ures +Ġlink ed +Ġdiscuss ed +Ġexpl o +Ġocc asion +ath an +Ġopp osite +Ġfac es +Ġden ied +ĠK night +Ġn ut +Ġapprox imately +Ġdisapp oint +onym ous +ĠB est +ĠL o +ĠH y +ĠA ff +Ġvot ing +an while +ĠII I +Ġinstit utions +ag ram +ĠD aily +Ġdr ag +Ġnear by +Ġgu ilty +Ġcon ver +P re +s hip +Ġre ward +Ġphilos oph +ĠS S +u gh +Ġapp s +f riend +Ġu pper +Ġad vert +Ġs now +Ġfr ust +Ġour selves +F r +ĠD ie +amp ion +Ġdis miss +Ġc ere +Ġsign al +f rom +Ġ ). +Ġ5 2 +Ġcr imes +it ors +est ival +use um +Ġcoun cil +ĠS aud +M ay +ĠG un +ic ian +et her +Ġsu fficient +ĠH en +so le +Ġhistor ical +ĠF ar +ĠT urn +Ġp in +Ġsuc ceed +m at +ly mp +Ġtrad ition +ĠO k +Ġc ro +Ġdesc ription +al le +Ġsk y +T e +Ġwide ly +Ġw ave +Ġdefin ition +ĠJew s +Ġcy cle +Ġref ere +Ġbr ings +us al +Ġal ive +Ġfrequ ently +Ġint ention +ĠCont rol +l v +y stem +Ġpriv acy +g ent +ren ce +ĠQu est +ĠChrist mas +Ġr ail +Ġco oper +Ġtest ed +ĠC apt +as ks +Ġcomfort able +Ġdel ivered +sc ape +Ġdep th +ĠG OP +Ġwrit es +Ġass ets +Ġsa v +im ents +Ġtrans ition +Ġart ist +ĠL ook +Ġl ob +Ġcomp onents +ar ity +Ġwalk ed +Ġro ot +Ġparticip ants +Ġnot iced +Ġres c +Ġn av +ĠAd minist +d a +ut ral +pl ate +Ġimport ance +Ġass ert +ious ly +c ription +Ġinj uries +ĠChe ck +Ġregist ered +Ġint ent +Ġmiss ed +ograph ic +Ġsent ence +oun ter +Ġassist ance +ev in +Ġdat abase +Ġbuild ings +Ġclass ic +Ġth inks +ĠOh io +P r +ug g +Ġfe e +p an +Ġeffect ively +Ġfac ility +Ġbe ar +Ġch apter +Ġdog s +ĠCol umb +Ġl atter +it ial +Ġad mitted +T V +ĠGe org +Ġpost s +\ \ +Ġlawy er +Ġequ ival +Ġm and +Ġcontro lled +ĠW alk +ĠAnd rew +Ġmen u +am ental +Ġprotect ed +v a +Ġadminist r +or al +Ġre in +ĠS ar +Ġamount s +Ġn ative +ĠM oon +Ġrep resents +Ġab andon +Ġcarry ing +Ġt ank +m ary +Ġdecl ared +T ube +Ġh at +Ġpun ish +el lect +m es +Ġun iverse +ĠR od +ph y +Ġinf rastructure +Ġ5 1 +Ġopp osed +ow nt +c a +ĠM ake +Ġhard ware +Ġco ffee +R el +b al +w orld +ĠS af +ĠSe a +in als +Ġown ed +Ġh all +ers ion +Ġdescrib e +ĠP ot +Ġport ion +Ġat mosp +Ġgovern ments +Ġdep ending +Ġoff ense +Ġtr ick +aw a +ĠL ine +ĠV is +ĠH ard +ĠOr ig +ĠCl ick +Ġdes k +ĠVal ley +ĠS ov +Ġmov ies +Ġrem ark +Ġm ail +Ġcons cious +Ġrul ing +ĠR ights +Ġmed ic +he nt +ĠW omen +> < +Ġrepl aced +ĠP rem +ĠTh anks +Ġre new +ĠB all +if orm +Ġsh ots +C omm +Ġar med +Ġconst ant +Ġt aste +Ġreal ized +Ġbu ff +Ġm o +Ġeffic ient +M ost +or ation +if ies +Ġcommun ication +Ġfl ood +Ġconsequ ences +Ġany way +ig g +ĠG M +ĠTh ank +Ġ iron +Ġev olution +ĠC op +tw itter +Ġ9 5 +Ġrelationship s +ad el +ĠYou ng +Ġpropos al +ay ers +uild ing +ĠH ot +OR E +c os +Ġcoll abor +P G +ax y +Ġknow ing +Ġsupport s +ow ed +Ġcontrol s +Ġmere ly +um er +Ġath let +Ġf ashion +p ath +Ġg ift +Ġer a +AN D +Ġkind s +ĠKore an +Ġleg it +ul ous +Ġess entially +Ġthe rap +n ic +Ġsuff ered +Ġh ur +Ġprom ise +Ġex cess +Ġover w +Ġpr ime +ĠH ouston +er ry +ĠM s +R S +201 2 +Ġst ores +ĠO lymp +Ġj ourney +Al though +S ub +ĠE duc +ĠCh apter +Ġrequest s +Ġconsum ers +Ġt iny +Ġis ol +ĠF air +b a +ĠY OU +Ġcr ash +ce ler +Ġemot ional +Ġgood s +Ġelect ed +Ġmod er +ĠLin ux +Ġbl ocks +Ġis land +ĠSoc iety +Ġelect ions +Ġbroad cast +Ġche ap +Ġn ations +Ġse asons +4 00 +Ġwas te +ĠS at +Ġfield s +em ploy +Ġprof ile +Ġauth ors +AL L +ĠG ra +w est +ĠT y +Ġdeath s +Ġv acc +Ġfor med +Ġd u +Ġon going +ĠMuslim s +el f +ig ure +Ġass ume +ĠUkrain e +w ater +Ġco ast +Ġvot ed +g or +ĠA S +ĠMich igan +az a +ĠAr m +i ro +Ġf lex +as ters +' ' +Ġwel come +ar l +Ġloc ations +ig ation +ĠF il +Ġbu ying +Ġarch itect +Ġhard er +ĠC ub +Ġinter face +Ġrestaur ant +Ġdisco ver +Ġex ceed +Ġfav our +ger y +Ġd uty +Ġp itch +ad or +ĠM ach +b oy +Ġrespond ed +Ġext ended +her s +M any +ra id +if er +ĠIn s +S er +Ġmed ium +s he +ĠS ports +Ġmag azine +ut ation +Ġlim its +ĠG all +Ġex ternal +raz il +Ġyoung er +t le +Ġrem ind +ĠC ON +Ġimmedi ate +Ġh idden +Ġvol unte +Ġsim pl +od cast +Ġph ase +d r +Ġpl ot +Ġexp osure +R I +og rap +v in +an ish +ĠAc ad +ĠEng ine +Ġexp ansion +ĠP ay +Y our +Ġpus hed +ĠE ll +ĠHe ad +Ġmarket ing +ĠA C +k et +Ġh its +Ġg ro +ĠA ge +ĠSc ot +] [ +Ġst im +Ġi Phone +Ī Ĵ +Ġn arrow +ĠGet ty +ĠTur key +Ġperfect ly +Ġen able +ut ch +Ġprec ise +Ġreg ime +Ġsh if +Ġcomp ens +g un +d iv +Ġch osen +ĠK en +An y +Ġtre es +Ġrecomm ended +ĠR en +u able +ĠH T +F ollow +E G +ĠH and +ĠK enn +Ġarg uments +Ġex ists +Ġb ike +ĠCons erv +Ġbre aking +ĠG ar +Ġc razy +Ġvirt ual +ay lor +ix el +Ġ19 80 +Ġper mission +ĠSer ies +Ġconsum er +Ġclose ly +c alled +Ġ5 4 +Ġhop es +Ġar ray +ĠW in +ĠLab our +Ġsp ons +ĠI re +Ġp ow +Ġread ers +Ġemploy ment +Ġcreat ure +Ġresult ing +Ġaccur ate +Ġmom ents +Ġarg ued +Ġp ed +D uring +Ġ5 3 +ĠT al +Ġs ought +Ġsuff ering +Ġ icon +le e +Ġ( $ +al ian + ° +Ġp ra +Ġbon us +( " +k o +Ġact ing +D E +f all +Ġcompar ison +Ġsm ooth +ĠN AS +u pp +ĠJose ph +ep ing +ĠT ake +ĠM id +Ġs ending +f ast +ĠF all +Ġdeal ing +us er +ĠOr gan +C o +Ġatt ached +Ġse es +% . +Ġtyp ical +AR T +Ġfind s +ĠAs ia +um in +ĠC ore +ĠE nt +in ent +u ce +ĠBl ood +ĠN ever +Ġem ails +Ġhigh light +Ġconf ront +at us +ut ed +Ġun us +Ġtop ic +ĠAd am +Ġb le +at i +Ġunder stood +S et +st ruct +T P +Ġm ob +a a +ĠSt art +pect ed +se ll +Ġded icated +ĠC A +u an +Ġsong s +esc ription +Ġte ch +Ġr ape +Ġas ide +Ġgr ant +Ġ5 6 +s ub +Ġarg ue +Ġcont aining +Ġsche dule +Ġliber al +Ġpublic ly +Ġheav ily +ĠU t +in er +ĠS ection +ĠC are +we et +l s +D is +âĶ Ģ +ĠF ollow +B ack +ĠI T +Ġb es +j i +ĠH it +est ed +Ġevery body +ĠSw ed +Ġfem in +Ġfac ilities +Ġcon ven +C omp +ĠO S +c ore +Ġan x +Ġdiv ision +ĠC am +ĠSt an +m ates +Ġexpl ore +pl om +Ġsh ares +pl oad +an es +Ġide al +et ers +ĠB ase +Ġpl astic +Ġdist inct +ĠNet work +ĠSe attle +Ġtrad ing +ens us +int end +Ġex hib +Ġinit ially +ĠF ood +Ġthous and +ĠBus iness +act er +Ġpar agraph +Ġrough ly +Ġw ww +Ġcreat ive +ĠCon f +Ġconsum ption +Ġfil ms +ag an +Ġob tain +Ġt all +Ġt or +Ġacknow led +Ġg rown +al o +K E +Ġ4 00 +end ers +t aining +U G +Ġsu icide +Ġwat ched +ĠL ist +al i +re hens +Ġsurround ing +Ġp ip +Ġf lying +ĠJ ava +ord an +Ġserv ing +in ations +p ost +Ġsh o +A v +Ġj ail +z y +Ġ199 9 +Ġ< / +Ġliter ally +ĠS ir +Ġexp osed +Ġl ies +st ar +Ġb at +Ġear ned +ĠD ig +Ġspec ified +ĠSe ason +Ġdeg rees +Don ald +Ġcent re +Ġsh aring +Ġwin ter +ĠC O +C he +Ġ Î +M P +Ġun w +Ġfew er +ĠM ir +Ġsomew here +ĠK ey +Ġattack ed +ĠK ir +Ġdom ain +Ġstrong er +Ġ9 9 +Ġpen alty +I d +Sc ript +Ġdecl ined +Ġne ck +Ġfra ud +Ġcur rency +Ġr ising +R C +â̦ â̦ +H z +Ġt ab +Ġtal ent +n am +ĠN BA +Ġvill age +Ġleg s +ĠN ext +E d +Ġac id +Ġhy d +8 00 +Ġinvol ving +ĠIm age +ĠBe fore +F l +Ġyes terday +S ource +Ġterror ist +Ġsu p +Ġsy nt +ĠSaud i +Ġw est +Ġr u +b urg +Ġvis ible +Ġstru ck +r ison +Ġaw esome +Ġd rawn +Ġansw ers +ĠG irl +ĠR am +Ġthreat s +Ġdef eat +os it +Ġv ent +atur ally +Americ an +end a +ĠH oly +Ġr um +% , +c ase +ĠHist ory +ĠYou Tube +Ġsit uations +ĠD NA +S te +Ġsa ved +It em +Ġrec ip +olog ist +Ġfac ed +Ġel ig +O nce +ĠL i +u h +Ġmist ake +ĠDiv ision +ĠB ell +Ġsympt oms + ® +Ġdom in +Ġfall ing +Ġend ing +as hes +Ġmat ches +ĠOn line +Ġexplan ation +D ef +red it +Ġany more +ĠT otal +ĠF OR +us hed +Ġlet ters +Ġris ks +ĠO K +Ġreported ly +: \ +Ġpl ate +Ġsubject s +Ġattempt ed +if ier +ian a +Ġunlike ly +ĠTh ough +um a +ĠIn vest +ĠPr in +ic an +ĠD ar +ĠColor ado +au g +Ġve get +a os +ri a +Ġshe l +Ġmark ed +Ġ( ) +Ġsp r +p o +ĠL ink +Ġdef e +ĠJ r +Ġthem e +Ġpass ion +ĠP en +Ġinf o +iz er +Ġsh it +ĠC ivil +ap se +c re +Ġpo ly +Ġcomp onent +ĠChar les +ĠIre land +ĠPro v +Ġdo ctors +Ġgr anted +Ġpain t +Ġhon or +Ġsm oke +Ġpay ments +Ġprim arily +ĠKing dom +r ich +ate ll +Ġde als +Ġsched uled +Ġfund amental +Ġprote in +Ġnewsp aper +Ġcl ients +yth on +ĠD ate +h us +Ġfeed back +Ġstret ch +Ġc ock +Ġhot el +ĠQue en +Ġsu gar +Ġj u +Ġmil k +Ġappro val +ĠL ive +Ġequival ent +ef ully +Ġins ert +z ona +Ġext ension +d ri +J ohn +Ġacc omp +S m +ĠF und +Ġconst antly +Ġ` ` +Ġgener ated +ĠA ction +ĠP sych +ĠT ri +Ġrecogn ize +Ġv ary +ph a +ĠR a +d f +et ch +ĠSov iet +Tw o +Ġpattern s +Ġprof ession +an ing +T ime +ĠL im +Ġcol ors +ĠA z +ĠT R +Ġinf ect +Ġphen omen +Ġshe ll +Al so +Ġput s +Ġdel ivery +Ġbro wn +Ġprocess ing +Ġlight s +ess age +ĠBro ok +ĠA ud +l ation +Ġindust rial +L ike +ĠB razil +rou s +ES S +ĠL uc +Ġsome how +Ġ8 5 +Ġpro port +Ġpolit icians +Ġindic ate +Ġh ole +Ġtechn iques +Ġcompet itive +Ġph r +Ġv o +ist ent +ĠD ream +Ġcamp us +Ġaspect s +Ġhelp ful +Ġsh ield +or se +Ġtrig ger +m al +Ġ5 8 +Ġt ort +Ġperson ally +Ġt ag +Ġkeep s +ĠV ideo +Ġben ch +Ġg ap +a ire +Ġe ast +Ġrec overy +per ial +Ġprof it +ĠM ic +Ġ5 7 +Ġcol on +Ġstrong ly +st yle +Ġalleg ations +h an +Ġrep orters +j o +r ine +arg et +and al +Ġ0 3 +Ġfl ash +tr ans +Ġstr ict +Ġpark ing +ĠPak istan +Ġl i +Ġwe ird +ĠE ric +Ġreg ions +ĠJ un +Ġint ellect +ĠW H +od ing +rib utes +up id +ĠT it +Ġf inger +or ia +Ġe lev +ĠF ield +Ġcon clusion +; ; +Ġfeel ings +Ġext ensive +Ġm ixed +Ġne uro +v y +Ġhar ass +ĠC irc +ou ch +Ġterrit ory +Ġsuccess fully +M ar +Ġing red +Ġoverw hel +Ġl ayer +V iew +Ġall ies +ill ance +ĠTh ree +Ġb unch +Ġnorm ally +Ġnet works +Ġsac r +ĠC IA +b les +Ġch ose +Ġopp onents +Ġregard less +Ġfr anch +Ġpre f +ĠP o +Ġbr idge +ann a +ĠSil ver +Ġw age +p age +ri or +Ġrad ical +ĠL ittle +Ġman ip +Ġsecret ary +Ġg ang +D R +F A +Ġdec ent +ĠSp irit +Ġun cle +ĠDevelop ment +Ġinvest ors +Ġwall s +Ġpub lish +Ġgener ate +iss ions +c ar +Ġprom ote +Ġcut ting +Ġche st +Ġdrink ing +Ġcollect ed +Ġ7 2 +Ġhop ing +Ġem br +gor ith +Ġwar ned +Ġinstruct ions +O G +ĠD id +ĠAg ency +Ġg ear +Ġcritic ism +ĠF urther +Ġut il +ann y +R ed +Ġcoun sel +ĠAs ian +Ġredu ction +p ool +Ġteach ing +Ġdeep ly +i y +Ġestim ates +Ġcho ices +Ġperman ent +in em +ke l +Ġf asc +p se +f ile +ĠL ow +ĠP erson +Ġt ournament +st al +Ġm el +U ST +ĠR ay +az i +V al +Ġcont ained +ĠH olly +Ġw ake +Ġreve al +Ġprocess es +ĠIS IS +Ġ0 9 +Ġbl ind +Ġste el +ĠB ad +Ġcare fully +app y +ro it +Ġg aming +Ġhous es +ĠC oll +Ġtr uck +er m +Ġsc ored +Ġocc as +ret urn +b ound +v ar +Ġsh arp +Ġaf raid +ĠE X +am ber +c ific +Ġsche me +N C +ĠPol it +Ġdecl ine +Ġ199 8 +Ġpus hing +Ġposs ession +Ġpriv ile +Ġteacher s +Ġy ield +H A +ĠDav is +it led +#### #### +Ġr ig +ĠD aniel +ac on +Ġh ide +ut en +Ġcolle agues +Ġprin ciples +Ġl oud +Ġs in +ĠDem on +Ġst one +Ġ0 2 +Ġt aught +Ġter rible +Ġst uck +ĠPol icy +te en +Ġimplement ation +ĠB BC +ĠAP I +Ġwhe el +all as +Ġch ampions +ol ars +play er +Ġrepeated ly +ĠSt ill +Ġlik es +ast y +es ter +ĠCath olic +R L +Ġb ath +Ġno ise +t itle +Ġn orthern +P art +Ġmag n +Ġf ab +ĠAs h +Ġdis pl +Ġtick et +Ġm urd +Ġalong side +ĠMus ic +Ġr iver +ĠSte el +ĠC L +ĠPl ayer +ĠM ult +ow ing +re p +s ize +Ġt ur +ĠGeorg ia +isc al +ra ction +Ġc able +Ġ5 9 +Ġw ins +Ġup coming +Ġsurv ive +Ġins pired +ĠEduc ation +Ġstat istics +ĠF oot +iam i +Ġy ellow +ĠP age +. - +ĠH as +Ġur ban +Ġa x +es sel +\ " +Ġquarter back +Ġreg ister +ĠLab or +Ġab ilities +ĠF amily +Ġvar iable +ĠPr ice +Ġcont em +Ġth in +ĠE qu +d ata +Ġg otten +Ġconst it +Ġas ks +Ġt ail +Ġexc iting +ĠE ffect +ĠSp anish +Ġencour age +ins on +ĠA h +Ġcommit ment +C S +Ġr ally +Ġ: : +Ġsubs id +Ġsp in +Ġcapt ured +201 8 +Ġinn oc +Ġalleged ly +ĠC ome +Ġart ists +ĠN umber +Ġelect ronic +Ġreg ional +ap es +Ġw ra +Ġmy th +pr ise +ĠM iller +ĠC reat +ĠEp isode +b ell +Ġdirect ed +Ġext ract +Ġs orry +Ġv ice +ag ger +ĠSu pport +Ġ6 6 +ĠI ron +Ġwonder ful +Ġg ra +N et +ion e +E ng +Ġsh ips +ik es +ĠK evin +it ar +Ġactiv ists +tr ue +ĠAri zona +ent h +ĠDes pite +ĠS E +Ġha bit +ern el +Ġin qu +Ġab ortion +Ġv oid +Ġexpl icit +Ġeng aged +Ġang ry +Ġr ating +Ġfr ag +b ro +ick ing +d ev +Ġwor ried +Ġob ser +Ġap artment +ĠG T +Ġest ate +ĠConst itution +em on +ĠS now +Ġcount y +Ġdis ag +ĠStep hen +Ġimm igrants +w ind +ĠN ations +Ġfol ks +O ut +Ġg all +Ġtarget ed +Ġst ead +ĠB on +ĠL ib +Ġinform ed +Ġ12 0 +ch ain +idel ines +or ough +Ġdri ven +Ġregular ly +Ġbas ket +Ġprinc iple +oc ument +Ġst un +ib ilities +ĠRom an +ĠAb out +Ġal ert +Ġdemocr acy +Ġrepresent ed +H S +c ers +p arent +Ar t +p ack +Ġdi plom +re ts +ĠN O +Ġcapt ure +ĠAd v +Ħ ¢ +Ġannounce ment +ĠL ear +Ġh ook +Ġpur s +ĠS uch +ĠC amer +Ġrefuge es +ĠV e +P ol +Ġrecogn ized +l ib +Ġhad n +A ss +Ġpil ot +us hing +Ġreturn ing +Ġtra il +ĠSt one +Ġrout ine +Ġcour ts +Ġdes per +Ġfriend ly +ĠIt aly +Ġpl ed +Ġbreat h +Ġstud io +N S +Ġimp ressive +ĠAfghan istan +Ġf ing +Ġd ownt +ink ing +ĠR og +i ary +col or +se x +ar on +Ġf ault +ĠN ick +D own +ĠR ose +ĠS outhern +X X +is odes +L ist +6 00 +Ġout come +er r +Ġelse where +Ġret ire +Ġp ounds +ĠGl obal +Pe ople +Ġcommun ications +Ġlo an +Ġrat io +ĠEm pire +Ġg onna +Ġinv ent +D F +Ġ19 70 +ĠComm on +p at +Ġprom ised +Ġd inner +ĠH om +Ġcreat es +Ġoper ate +ver ty +ĠJ ordan +et ime +Ġsust ain +R eg +Ġincred ible +im a +Ġwar rant +Ġm m +A tt +Ġlaw suit +Ġreview s +it ure +ĠS ource +l ights +ĠF ord +Ġ6 3 +g roup +st ore +Ġfeat ured +Ġfore ver +Ġpo verty +ĠP op +ĠC NN +az z +ab is +ach ing +Ġl aid +ĠSu pp +Ġfil ter +en a +ĠCommun ity +Ġcreat ures +u ction +ĠR oyal +Ġassoci ation +ĠCon nect +ĠBr ad +âĸ Ī +l ers +the re +ĠG i +Ġval uable +AC K +ĠT aylor +Ġl iquid +ĠAtt orney +ĠCar l +ĠF inal +ag a +ĠWil son +B ecause +ĠProf essor +ak a +Ġincred ibly +r ance +! ) +R ef +s k +Ġsol utions +Ġatmosp here +Ġbl ame +um es +ĠN ob +C A +um ps +r ical +ĠPut in +ĠD est +or ic +ĠP A +Ġrespect ively +w an +Ġfif th +â Ħ¢ +ĠC ry +Ġgovern or +res ident +Ġpurch ased +Ġh ack +Ġint ense +ob s +Ġorig in +Ġdef ine +Ġcare ful +** * +Ġshould er +Cl ick +Ġt ied +Ġdest ruction +ou red +Ġno body +Ġh o +ĠEx per +Ġt ip +" ; +Ġtechn ique +Ġj ur +ĠP ok +b ow +Ġleg end +Ġacc ord +Ġbus y +ĠInt el +Ġh ang +ak i +. ] +âĢĶâĢĶ âĢĶâĢĶ +Ġsur gery +Ġrep rodu +Ġun iform +Ġscen es +c ode +Ġ6 2 +l isher +ĠH ave +ph ia +Ġcry pt +Ġrec on +Ġsc ream +Ġadop ted +Ġsc ores +N e +ĠIt alian +in cluding +B O +Ġindic ated +Ġent ertain +G u +T ext +i el +Ġtw enty +Ġeng age +off s +ĠPac ific +Ġsm ile +Ġperson nel +Ġto ler +Ġdo ors +Ġt one +Ġmach ines +Ġent ering +ten ance +C O +ĠJer sey +Ġfore st +Ġhor se +Ġcompl aint +ĠSpr ing +y o +ĠPl us +ed ing +ĠRet urn +qu arters +ial s +c ow +Ġacad emic +Ġf ruit +Ġ199 6 +og ether +Ġw ine +Ġpur su +ĠSte ven +Ġlic ens +Wh o +Ġclot hes +re ction +Ġsqu ad +Ġst able +Ġr aw +z ens +St ar +ut ies +anc er +Ġke ys +ĠM u +Ġcompl icated +ig er +ĠTe xt +Ġabs or +Ġ6 8 +Ġfun ny +Ġrel ief +ĠL ew +ĠC ook +Ġch art +Ġdraw ing +G E +Ġmod ule +ĠB ull +I LL +Ġs alt +0000 0000 +il le +Ġres ource +aw ay +adel phia +ĠB ru +Ġ6 7 +Ġsome body +Ġparticip ate +Ġro se +we red +Ġmus cle +Ġcons ent +Ġcontin uing +ĠGuard ian +ĠOr der +reg on +Ġre ar +Ġprov ision +Ġlik ed +ri ent +Ġb ra +Tr ans +Ġmeet ings +Ġto x +Ġcon vent +Ġaut o +Ġrec ording +ĠSo ft +00 1 +ĠR oll +Ġprogram ming +Ġp ic +Ġprov ed +Ġst ab +ĠA st +Ġca ption +ul ating +ĠAtt ack +Ġnew ly +Ġ199 7 +f r +Ġdis cipl +ĠGree k +Ġed ition +ĠDo es +ĠB ox +if le +ack et +Ġpass es +Ġgu est +Ġac celer +it als +U D +Ġaut hent +ĠR est +ov al +t a +u ine +Ġarm or +ĠT own +Ġcomp at +Ġinc hes +Des pite +Ġass ign +he rent +Ġprep are +ĠM eg +oc key +Ġdep ends +Ġtrack s +w atch +Ġl ists +ĠN orthern +Ġal ter +re c +ĠE astern +Ġcond em +Ġevery where +? ' +Ġaff ili +Ġf ought +": {" +Ġm ac +it arian +Ġsc ope +ĠA L +aw s +ar ms +Ġqu e +Ġenjoy ed +nes ota +Ġagg ressive +ĠSt ory +ĠI V +Ġrec ipe +Ġrare ly +ĠMed ical +val ue +ang el +ay ing +omet hing +Ġsub section +Ġs outhern +Ġfrequ ency +re te +roll ed +ult s +ĠN ic +Ġbeh alf +Ġsequ ence +ab et +Ġcontrovers ial +Ġcomp rom +Ġwork er +Ġmain ly +Ġal gorith +ĠM ajor +or ce +g ender +Ġorgan ized +Ġf ake +Ġconclud ed +ĠE D +ĠEx ec +r age +Ġch ances +ber ry +ĠTr ad +Ġconfig uration +Ġwithd raw +Ġf ro +ud es +ĠBro ther +ĠB rian +Ġtri es +Ġsam ples +Ġb id +ĠGold en +Ġphot ograph +if est +ĠD O +ĠPar liament +******** ******** +R em +Ġcont est +Ġsign ing +p x +ĠZ eal +âĶĢ âĶĢ +E ar +Ġex it +Be fore +ĠCor por +n ull +mon th +Ġrac ial +ott ed +ĠV eg +ĠRe uters +Ġsw ord +ps on +ĠRom ney +a ed +Ġt rib +Ġin ner +Ġprot ocol +ĠB i +ĠM iami +ever al +p ress +Ġsh ipping +ĠAm endment +ĠHow ard +con nect +ĠD isc +ĠJ ac +iam ond +ĠThere fore +s es +ĠPrin cess +ĠUS B +ĠAn th +Ġsurve illance +Ġap olog +Ġ6 1 +ow a +Ġf ulf +j s +Ġl uck +ust ed +Ġ § +n i +Ġant icip +em an +Ġwin ner +Ġsil ver +ll a +ic ity +Ġunus ual +Ġcr ack +Ġt ies +e z +Ġpract ical +Ġprov ince +ĠPl ace +Ġprior ity +IC E +Ġdescrib es +Ġbr anch +F orm +ask a +miss ions +b i +Ġp orn +ĠTur k +Ġent hus +Ġf ighters +Ġ0 8 +ĠDet roit +Ġfound ation +av id +A re +Ġjud gment +cl ing +Ġsol ve +ĠDes ign +W here +hes is +ĠT ro +a fter +Ġne utral +ĠPalestin ian +ĠHolly wood +Ġadv is +ĠN on +y es +ol is +Ġrep utation +Ġsm ell +Ġb read +ĠB ul +ĠBe ach +Ġclaim ing +Ġgen etic +Ġtechn ologies +Ġupgr ade +row s +Ġdevelop er +ĠJ osh +ĠDis ney +erv ed +ip al +Ġun ex +Ġbare ly +t hen +ĠP ub +Ġill ness +et ary +ĠB al +Ġp atch +Ġbut t +Ġst upid +ĠD og +ĠD allas +f ront +ie ce +Ġprot ests +Ġch at +oen ix +Ġw ing +Ġpar liament +Ġ7 7 +ose xual +Ġre nder +pt ions +ĠCo ast +os a +ĠG reg +h op +ĠMan agement +Ġbit coin +Ġrec over +Ġincor por +or ne +ĠUs ing +Ġpre ced +Ġthreat ened +Ġspirit ual +ĠE vent +ĠF red +Ġadvert ising +Ġimprove ments +ĠC ustom +Ġer rors +Ġsens itive +ĠN avy +Ġcre am +L ook +Ġex clusive +Ġcomp rehens +Ġde leg +Ġcon ce +Ġrem em +Ġstruct ures +Ġst ored +N D +Ġ1 000 +U P +ĠB udd +A F +w oman +ĠAcad emy +ð Ł +se a +Ġtem porary +Ab out +es ters +Ġtick ets +Ġposs ess +in ch +o z +Ġl a +Ġcontract s +Ġun p +Ġc ig +ĠK at +ult ural +as m +Ġmount ain +ĠCapt ain +St ep +m aking +ĠSp ain +Ġequ ally +Ġl ands +at ers +Ġreject ed +er a +im m +ri x +C D +Ġtrans action +g ener +less ly +Ġ| | +Ġc os +ĠHen ry +Ġprov isions +Ġg ained +Ġdirect ory +Ġra ising +ĠS ep +ol en +ond er +Ġcon sole +in st +Ġb om +Ġunc ertain +1 50 +ock ing +Ġmeas ured +Ġpl ain +Ġse ats +Ġd ict +S L +af e +Ġest imate +iz on +at hered +Ġcontribut ed +Ġep isodes +omm od +G r +AN T +Ġ6 9 +G ener +Ġ2 50 +vious ly +rog en +Ġterror ism +Ġmove ments +ent le +oun ce +ĠS oul +Ġpre v +ĠT able +act s +ri ors +t ab +Ġsuff er +Ġn erv +Ġmain stream +ĠW olf +Ġfranch ise +b at +Ġdem ands +Ġag enda +Ġdo zen +Ġclin ical +iz ard +ĠO p +t d +Ġvis ited +ĠPer haps +Ġact or +Ġde lic +Ġcont ribute +Ġin ject +ĠE s +ac co +Ġlist ening +Ġcon gress +epend ent +Ġprem ium +Ġ7 6 +ĠIr ish +Ġass igned +ĠPh ys +Ġworld wide +Ġnarr ative +ot ype +m ont +b ase +ĠB owl +ĠAdminist ration +Ġrel ation +ĠE V +C P +Ġco vers +Ġ7 8 +Ġcert ific +Ġgr ass +Ġ0 4 +pir acy +ir a +Ġengine ering +ĠM ars +Ġun employ +ĠFore ign +st ract +Ġv en +Ġst eal +Ġrepl ied +Ġult imate +Ġtit les +d ated +Ġj oy +a us +Ġhy per +ak u +Ġoffic ially +ĠPro duct +Ġdifficult y +per or +Ġresult ed +rib ed +l ink +wh o +~~ ~~ +ĠSpe ed +ĠV iet +W ind +ĠBar ack +Ġrestrict ions +ĠSh are +Ġ199 5 +ition ally +Ġbeaut y +op t +Ġm aps +ĠC R +ĠN ation +ĠCru z +W ill +Ġelectric ity +Ġor g +Ġb urd +Ġviol ation +Ġus age +Ġper mit +ĠCh ron +ĠF ant +Ġn aturally +Ġ0 7 +Ġth rown +ĠAw oken +Ġal ien +ĠHer o +ĠK ent +ĠR ick +ri ke +Ġp ace +}, {" +G L +Ġpo ison +ĠT ower +Ġform al +al ysis +Ġgen uine +Ġk il +a ver +Ġproced ure +ĠPro p +intend o +ĠM ain +as ant +Ġtr ained +G ame +ĠL oad +ĠM A +Ġcru cial +Ġle ts +ĠF R +Ġch ampion +1 01 +ĠCon ference +Ġwrit ers +Ġconnect ions +Ġo kay +ir ms +ĠR and +Ġenc ounter +ĠB uff +Ġachie ved +Ġche cks +isc ons +Ġassist ant +Ġwhen ever +ĠA ccess +ĠU r +b in +Ġcl ock +is p +op her +Ġb orrow +Ġm ad +Ġperson ality +on ly +IS T +ab ama +Ġg ains +Ġcommon ly +Ġter r +Ġhyp ot +Ġre ly +Ġt iss +iscons in +Ġrid ic +f unction +ĠO regon +Ġun com +r ating +el and +ĠN C +Ġm oon +ann on +Ġvulner able +ut ive +³³ ³³ +ĠRad io +Ġw estern +se ct +ĠT ony +Ġocc urs +ĠO s +ĠH on +Ã Ń +Ġv essel +ĠScot land +Ġdiscrim ination +Ġsubsequ ent +st ring +Ġfant asy +ĠSh adow +Ġtest im +W E +it i +r as +Ġbo at +Ġmar ks +Ġord inary +Ġre n +Ġrepresent ative +Ġpet ition +Ġ7 3 +Ġad venture +Ġign ore +ĠPhil adelphia +ĠS av +V P +Ġfact ory +Ġt asks +Ġdep ression +z ed +................ ................ +ĠSt orm +Ġc ogn +Ġelig ible +Ġredu cing +v ia +Ġ0 5 +Ġstri king +Ġdoll ar +h o +O V +Ġinstr ument +Ġphilosoph y +ĠMo ore +ĠA venue +Ġrul ed +ĠFr ont +IN E +ĠM ah +Ġscen ario +ĠNAS A +Ġen orm +Ġdeb ut +Ġte a +T oday +Ġabs ence +S im +Ġh am +le ep +Ġt ables +ĠHe art +M I +K e +re qu +V D +m ap +Ġchair man +Ġp ump +Ġrapid ly +v i +Ġsubstant ial +E P +d es +ch ant +ili pp +ĠS anta +ri ers +anche ster +L oad +ĠC ase +Ġsa ving +Ġ7 4 +ĠA FP +er ning +oun ced +ĠMin nesota +ĠW as +Ġrec ru +Ġassess ment +ĠB ron +U E +Ġdynam ic +Ġf urn +ul ator +Ġprop ag +h igh +Ġacc ommod +Ġst ack +ĠS us +w rit +Ġre ven +ĠGod d +ĠZeal and +ab s +Ġbr ut +Ġper pet +h ot +Ġhard ly +ĠB urn +ãĤ ¹ +Ġst y +Ġtrans actions +Ġg ate +Ġsc reens +Ġsub mitted +Ġ1 01 +Ġlangu ages +ugh t +em en +Ġfall s +Ġc oc +Ĥ ¬ +Ġstri kes +p a +Ġdel iber +ĠI M +Ġrel ax +ann els +ĠSen ator +Ġext rem +Ġ} , +ĠDe b +Ġbe ll +Ġdis order +c ut +Ġi OS +Ġl ocked +Ġem issions +Ġshort ly +" ] +ĠJud ge +ĠS ometimes +Ġr ival +Ġd ust +Ġreach ing +F ile +¯¯ ¯¯ +ino is +ĠJ ason +Ġs atell +are t +Ġst ations +Ġag ric +ĠTechn ology +com es +ĠUn fortunately +ĠChild ren +Ġappl ies +ast ed +Ġan ger +ail ability +ĠDam age +Ġcomp are +ĠStand ard +Ġaim ed +ĠB a +angu age +Ġreg ulation +Ġj ury +Ġair port +Ġse ctions +ĠPr ince +em ed +Ġmedic ine +Ġh itting +Ġsp ark +ol ves +Ġad s +St ate +Ġfood s +Ġrepl acement +Ġch icken +Ġlow est +Ġmind s +Ġinvol ves +u i +Ġarr ang +Ġproced ures +ĠWh ich +ivers ary +Ġb ills +Ġimprove ment +Ġin ev +Ġexpect ations +Ġintellect ual +Ġsp aces +Ġmechan ism +2 50 +bre ak +ĠZ e +ĠT enn +ĠB alt +Ġbar rel +Ġstat ic +man n +Pol ice +Ġt ips +Ġhand ling +c us +od ed +il ton +ir y +Ġjournal ists +our se +Ġcom ic +Ġnom ine +IT Y +Ġvers us +Ġlo op +Ġsur f +ĠInd ust +ĠHun ter +Ġbelief s +is an +Ġset up +Ġbre w +im age +Ġcomput ers +f ol +} ," +ĠMed al +Ġtax p +Ġdisplay ed +Ġg rav +Ġf iscal +M on +ĠMos cow +ĠK ong +ĠCent re +Ġcamer as +ĠMr s +ĠH ay +Ġa ver +ĠK elly +p y +Ġrequire ment +Ġent itled +omb ie +Ġsh adow +ag ic +ĠA k +Ġel ite +Ġdiv ided +Ġhead ing +Ġcop ies +Ġloss es +Ġv it +k ed +ĠB ry +Ġan s +ĠSte am +Ġrep orter +he im +ĠIt em +Ġsuper ior +d on +ere nt +à ¶ +Ġtherap y +Ġpe ak +ĠMod el +Ġl ying +Ġg am +z er +r itten +Ġrespons es +Ġconsider ation +ĠB ible +Ġl oyal +Ġinst ant +Ġp m +ĠFore st +à ¼ +Ġext end +Ġconv icted +Ġfound er +Ġconv in +ĠO ak +che ck +Ġsch olars +p ed +Ġover se +T op +c ount +ĠAr k + · +Ġ0 6 +ĠL A +m d +ĠLat in +im ental +ĠC PU +Ġsubst ance +Ġminor ity +Ġmanufact uring +E r +ocol ate +Ġatt ended +ĠMan ager +r ations +Ġappreci ate +om y +GB T +id ency +B L +Ġguarant ee +pos ition +Ġo cean +clud e +Ġhead ed +Ġt ape +Ġlo ose +Ġlog ic +Ġpro ven +Ġsp ir +Ġad mit +is a +Ġinvestig ate +Ġ199 4 +sy lv +ĠL ost +c est +Ġ7 1 +Ġrequest ed +Ġwind ows +ĠPok é +ĠWith out +M et +Ġbehavi our +Ġread er +Ġh ung +ĠKe ep +Ġro les +Ġimplement ed +Ġbl ank +Ġserv es +ĠJ ay +Ġc ited +ĠF riend +prof it +ap on +Ġrep air +it em +arr ass +Ġcrit ics +ad i +ĠF ather +Ġsh out +Ġf ool +Ġ8 8 +Ġprodu cing +Ġl ib +Ġround s +Ġcirc le +Ġpre par +Ġsub mit +Ġn ic +mor row +ãĥ « +U nder +Ġv ital +ater n +Ġpass word +Ġpublic ation +Ġprom inent +Ġspeak s +Ġb ars +Ġde eper +ĠM ill +port ed +Ġw id +Ġbut ter +Ġsm oking +Ġindic ates +K ey +rop ri +ĠF ile +all ing +ast ing +ĠR us +Ġad j +Ġ7 9 +av al +Ġpres um +bur gh +on ic +Ġf ur +Ġpoll s +ik a +Ġsecond ary +Ġmon ster +ig s +ĠCur rent +E vent +Ġowners hip +end ar +Ġarri ve +ĠT ax +Ġn ull +ĠPri v +Ġth ro +Ġk iss +c at +Ġup set +ang le +it ches +ect or +olog ists +ĠGal axy +Ġcor ruption +Ġh int +ent er +ĠH ospital +Ġgreat ly +Ġbeg un +es y +Ġso il +ĠAnt on +Ġmain tenance +ãĥ © +Ġdo zens +Ġhuman ity +ĠAl abama +Ġr om +w orth +ap ing +sylv ania +l ah +Ġg athered +G A +Ġattack ing +f ound +ĠSqu are +Ġar bit +ict ions +ĠW isconsin +Ġd ance +ĠS aint +arch y +Ġbase ball +Ġcontribut ions +Ġliter ature +Ġex ha +per ty +t est +Ġb ab +Ġcontain er +let ter +Ġfall en +Ġwebs ites +Ġbott le +ĠS ac +Ġbre ast +ĠP L +Ġveter an +Ġinterview s +ĠA le +Ġb anned +eng ers +ĠRev olution +in th +Ġconc erning +IV E +Ġexp enses +ĠMatt hew +ĠColumb ia +d s +ist ance +Ġent ity +.. ." +Ġrel iable +Ġpar alle +ĠChrist ians +Ġopin ions +Ġin du +l ow +Ġcompet e +Ġth orough +Ġemploy ed +Ġestablish ment +ig en +ĠC ro +Ġlawy ers +ĠSt ation +T E +ĠL ind +ĠP ur +it ary +Ġeffic iency +âĢ IJ +ĠL y +Ġm ask +Ġdis aster +Ġag es +ER E +es is +ĠH old +Ġcas ual +b led +Ġen abled +ĠEn vironment +ĠInt elligence +i per +ĠM ap +ĠB E +Ġemer ged +is dom +Ġc abin +Ġregist ration +Ġfing ers +Ġro ster +Ġfram ework +ĠDo ctor +et ts +Ġtransport ation +Ġaware ness +H er +Ġattempt ing +O ff +ĠSt ore +ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ +ĠK now +Ġdef ence +Ġsc an +ĠT en +ĠCh air +ĠP H +ĠAtl anta +Ġfuck ing +Ġans wered +b n +ĠK ar +Ġcateg ories +Ġr ational +Ġc ust +Ġrob ot +Ġcorrect ly +Ġg if +Ġgraph ics +m ic +Ġground s +ĠO pp +i ate +Ġdist ributed +Ġsan ctions +Ġchalleng ing +ut o +Ġingred ients +Ġinv ited +Ġfound ed +ĠRe qu +d ed +Ġb owl +Ġbrother s +ĠH a +I O +Ġw ages +im ore +oc ial +Ġse ed +ative ly +Ġaddress es +ĠI owa +ab eth +Ġatt itude +is d +ch ild +Ġm ole +Ġdisco very +y ard +B r +Ġ8 2 +Ġsuppl ies +ell ing +Ġdist ingu +C R +Ġre cept +Ġ vert +Ġsw im +b ec +d oor +ĠY eah +Ġg al +Ġinter act +ĠE SP +ĠC S +amp s +Ġconvin ced +Ġobject ive +Ġdis h +ĠPhot os +l ad +Ġdownt own +o il +in ction +Ġto morrow +ĠC OM +Ġsurv ival +sh ot +Ġsett lement +C ons +ĠX box +int erest +ĠS M +arg o +en ess +Ġeth nic +b ered +M in +ĠT ok +Ġinc ent +ĠComm and +Ġmain tained +Ġbreak s +br idge +at ar +ag g +ĠF inally +un icip +ĠO nt +le ft +Ġrecogn ition +Ġ* / +ĠP ers +Ġwe lf +Ġaddress ed +ĠK ansas +Ġvir us +Ġwhere as +Ġp apers +ram s +ĠMin istry +Ġple asure +Ġacqu ired +Ġd uration +j pg +Ġcal m +ĠN HL +Ġburn ing +Ġfold er +ick ed +ĠP y +ĠIll inois +Cl ass +ĠGodd ess +Ġperform ing +Ġwelf are +j ar +In ter +Ġl in +Ġenh ance +Ġnot ion +f are +yp es +ĠAre a +Ġcann abis +ĠDie go +f s +ĠM anchester +com m +in ite +Ġcover ing +ĠS ound +Ġ19 60 +Ġ8 4 +e lect +z ing +Ġcitiz en +Ġph ones +Ġr aid +Ġign ored +ĠOb ject +Ġu pload +c ard +Ġmod ified +Ġroom s +ia h +r ange +he ast +ach us +Ġsuggest ing +âĢ ĭ +gr ade +E l +Ġclot hing +Ġr h +ĠH an +un ity +en cing +ĠAust in +sec ution +t ra +d em +ĠQ ual +Ġhe aven +Ġst ages +Ġw edd +pl us +ific ial +ĠIm m +ĠH o +iet ies +Ġphr ase +Ġbr ill +act ory +Ġprov iders +Ġsil ence +Ġa er +ĠA I +ĠAd venture +Ġplatform s +Ġdemonstr ated +Ġinter f +ing ton +Ġr aces +Ġgr ade +ult ane +ĠTh rough +f alse +Ġb ow +ĠA B +Ġfl avor +Ġhistor ic +g ov +Ġcol our +Ġview ed +ĠEm ail +el come +Ġinter vention +Ġd iversity +Ġperiod s +Ġre verse +ĠV ery +Ġqu ote +ĠLe ft +th rough +Ġsc rew +Ġland ing +Ġp ill +Ġw et +Ġprot esters +Ġrepe at +av ed +er k +Ġsal ary +ĠPenn sylvania +St ill +Ġmay or +Ġkit chen +Ġfeat uring +ĠM useum +ĠT ournament +ĠF al +Ġser vers +U C +Ġany body +im g +ĠTr ade +ixt ure +the less +Ġfin ance +Ġcl osing +ĠPat ri +i ac +ab el +Ġ> > +or ous +Ġf irms +sc reen +un a +Ġemb arrass +ul se +Ġlet ting +Ġth rew +ile y +Ġch annels +l an +ĠVeg as +Ġse ar +Ġfant astic +ar re +uzz le +ĠD er +Th ose +Ġsw ing +Ġshe et +ind ex +co ver +og an +Ġvari ables +ĠTe ch +Ġsp oken +ac hel +ĠD a +ĠMount ain +Ġload ed +Ġfoot age +vers ion +Ġun l +ĠPh oenix +Ġthrow ing +Ġf iring +Ġtrack ing +Ġw idth +Ġstrugg ling +ro oms +ot ion +Ġmonth ly +ĠSer ver +Ġegg s +op en +M C +Ġ199 3 +Ġh ired +Ġstay ed +ĠAll en +Ġst ro +Ġ9 8 +st ep +ĠTurk ish +Ġfab ric +ist ing +ĠD om +Ġd ates +Ġpr on +Ġbasket ball +Ġl ucky +ĠArab ia +Ġassum ed +est y +Ġaff airs +Ġgl ad +ĠInd eed +ĠF A +ĠW ord +Ġjo ining +if ice +p read +ir ts +ĠSe lect +Ġpop ulations +aw are +Ġn ose +Ġcompl aints +st art +Ġsc oring +Th anks +Ġmin ing +Ġvisit ors +S H +Ġdam aged +Ġcharacter istics +ĠP ent +D C +Ġ8 3 +ĠS ix +r ates +Ġfl ags +ĠB rew +d og +M ark +// // +Ġexec ution +Ġj oke +ph ones +Ġtestim ony +Ġob st +Q L +ĠC ut +Ġstud ied +ĠN intendo +ick et +ĠN BC +Ġl ad +ĠB ra +ĠM oh +Ġk ernel +Ġoverwhel ming +Ġag ed +Ġapplic able +ĠC ond +Ġroad s +ĠBl ock +m ade +od ge +Ġcomm ands +Ġoff ices +vel and +Ġt ut +Ġrece iver +ĠF ro +Ġsho pping +Ġi P +ĠSt re +ĠA BC +Ġentertain ment +ĠB ow +ort ed +M c +Ġread s +gr ad +ĠCol lect +Ġâ ĪĴ +ĠCap ital +eder ation +Ġemploy er +Ġinvolve ment +Ġanx iety +al ia +Ġro of +ĠAm ong +ĠDemocr at +Ġstat s +ĠV ill +Ġconst itutional +Ġrefer ring +itt y +Ġtack le +out ube +Ġback ed +ĠH ong +ĠBro ad +Ġe le +ĠO tt +Ġ199 2 +h our +achus etts +C al +Ġdefe ated +Ġ8 1 +es p +Ġseem ingly +w as +ĠJ enn +ĠK urd +Ġg ene +Ġdisc ount +R et +EC T +( ); +Ġclub s +Ġs id +ĠM arsh +Che ck +Ġp p +ĠE ag +ides pread +Ġbe ings +F T +Ġintrodu ction +ĠCh ange +AR D +Ġ1 10 +ad ows +ier ce +Ġme al +a uthor +ĠB ang +lah oma +Ġr anks +201 1 +?? ?? +m ax +Ġcoll apse +Ġop ens +Ġe cho +Ġs oph +Ġrac ist +Ġenorm ous +Ġw aves +Ġt ap +Ġcomprehens ive +. -- +ĠR oy +Ġfarm ers +Rel ated +a ired +ron es +ĠC rim +Ġproport ion +Ġdesign s +Ġnegoti ations +Ġvirt ually +ĠBat man +Ġwar n +Ġlegit imate +m ate +Ġcon vention +, , +net ic +ĠS D +Ġconsist ently +Ġcompens ation +Ġpunish ment +Ġy e +Ġt ie +ĠB ureau +ir lf +ĠB u +ĠA ren +ĠPh ilipp +Ġkn ife +Ġmem ories +ĠR oss +Ġang le +Ġ8 6 +ĠTh under +Ġre nd +ĠT our +Ġcount s +s ung +ĠIm p +Ġeduc ational +Ġaccess ible +C OM +Ġd rew +y er +G l +am ine +OR T +O B +I B +m aster +Ġtri als +og y +h ar +ĠTr ust +Ġprefer red +irlf riend +ĠN ev +Ġb in +Ġc ow +P age +Ġsign ature +ĠB L +7 00 +Ġret ired +Ġby tes +Ġneigh b +ĠLeg end +Ġdev ast +Ġsuspect ed +is ons +ĠPoké mon +sc ale +Ġcap abilities +Ġre vel +Ġche ese +d y +igr ant +Ġfail ing +b its +ĠHer oes +ĠG host +ĠS cient +Ġappoint ed +ur i +Ġinst itution +Ġexpand ed +g reg +Ġmonitor ing +Ġp odcast +Ġcoal ition +Ġ9 6 +J o +Ġst olen +ĠS ab +Ġstop s +Ġhol iday +Ġint r +C ar +Bl ack +ĠL GBT +Ġwar ming +ĠAnd erson +Ġ8 9 +Ġprodu cer +M ed +Ġaccur acy +ĠMar vel +iz abeth +ĠPat rick +m ony +Ġmin i +ac les +Ġover t +the y +Ġmembers hip +ĠV en +Ġex ch +Ġrem oval +ĠD ave +T Y +m ad +ĠF ind +Ġad equ +Ġe c +Ġte eth +Ġemot ion +Ġper m +Ġsole ly +d b +Ġextra ord +IG HT +c al +Ġgu idelines +Ġd ying +Ġsusp ended +ĠPrem ier +ĠAnth ony +el ve +Ġd ad +ĠE th +ĠFoot ball +Ġabandon ed +Ġ< < +Ġm arch +Ġhor ror +â̦ " +Ġchild hood +Ġcampaign s +Ġl unch +ĠAl bert +bl ock +âĸĪ âĸĪ +ound ing +Ġb one +or gan +ad ers +ĠFl ash +ĠDri ve +Ġton ight +Ġw ars +ĠF L +Ġform ation +con st +New s +Ġcom pe +or ious +ĠSt aff +Ġdiscuss ions +ĠProt ection +ĠJ am +Ġcrit eria +Ġinstall ation +Ġaccompl ish +iz za +Ġpub lisher +Ġresc ue +ĠT ry +U LL +ĠS om +ĠH op +ore t +th s +ord on +Ġp ocket +ĠIn v +Down load +ĠCr ime +Ġb ene +ĠGu ide +ĠAs sembly +Ġparam eters +I E +ĠAlex ander +Ġconc ert +ĠSc he +Ġsh oes +Ġvis iting +Ġrec all +Ġb ub +Ġr ural +Ġconc rete +ĠR os +N ext +R uss +Ġlo ans +ĠSh ield +Ġtre m +hem at +k g +ĠHar ris +is ition +ĠM ove +ĠF C +Ġf ate +ĠCh o +Ġt ired +Ġprinc ipal +h ist +ien ces +ath y +Ġse vent +Ġm ood +Ġstrateg ic +Ġdise ases +Ġfor um +Ġtem por +Ġhead quarters +P ar +ig e +fl ix +Ġgu itar +Ġ9 4 +On ly +Ġrele ases +ro ph +================ ================ +Ġ6 00 +ĠContin ue +ig ate +ĠC rit +sy stem +Ġdis abled +Ġunex pected +ith ub +Ġuncle ar +ĠE st +Ġcontr ad +Ġstrateg ies +vent ures +Ġpass age +AM E +Ġimpro ving +Ġreve als +Ġdecre ase +ov a +Ġann oy +ĠSh ort +ĠL ibrary +Ġcy ber +n ell +ĠH ur +ĠC B +Ġphot ograp +U I +Ġs ed +G e +Ġ8 7 +Ġd iverse +Ġencour aged +Ġcons piracy +Ġbird s +Ġoper ator +Ġhand ful +Ġclass ified +? ) +Ġdram atic +Ġinvestig ators +it o +Ġw idespread +ĠR oom +-------------------------------- -------------------------------- +Ġcollect ive +Ġjournal ist +St ring +Ġtemper atures +il a +Ġgu id +Ġins pect +Ġmiss ile +ĠMay or +Ġman ual +Ġsim ultane +Ġrat ings +Ġsu ck +Ġ9 7 +Ġunivers al +Ġph arm +Ġdis rupt +ian o +A V +Ġf t +Ġstat ist +old s +ĠWalk er +ph p +Ġunder t +ĠL as +ish op +nt il +res hold +ĠWhe ther +M s +Ġden y +ĠCl oud +Ġprov ider +Ġsurv iv +ĠUp date +h as +Ġmist akes +ch arge +pl ed +r ity +Ġn ode +ĠMass achusetts +ool s +lic ation +Ġf ails +em ale +or i +back s +Ġsh irt +Ġ' ' +ĠN AT +Ġwat ers +els on +Ġe ase +Ġsc ar +Ġcont ents +m ind +Ġcont ribution +Ġsh r +Ġhand ed +Ġst ability +Ġtra ve +E m +Ġmir ror +12 3 +Ġwe igh +Ġf iction +ou ver +ist ant +r ition +ĠF ed +Ġphys ically +Ġst ake +ĠArt icle +ĠAr c +ĠLew is +ĠM ind +Ġdemonstr ate +Ġprof its +v ision +om ic +ol id +Ġbatt les +Ġdri ves +Ġeas tern +ĠS ony +!! ! +ar ation +v ard +ĠG L +port ation +Ġ9 2 +Ġlaw makers +Ġprotect ing +ĠE PA +Ġy eah +Ġsh ame +ol ph +e ven +x it +Ġatt ach +Ġrepresent ing +Ġob s +ĠUt ah +iff s +ĠFre edom +à ³ +A K +Ġinc idents +it age +Ġview ers +c d +Ġm ouse +Ġcl ar +Ġaccord ance +Ġb ot +c or +ĠSum mer +he ld +Ġinnoc ent +Ġiniti ative +ol s +________________ ________________ +Ġsp ots +p ace +Ġconvent ional +Ġcorpor ations +Ġblock ed +H D +at tered +Ġref ers +Ġbu ck +ĠDig ital +12 0 +Ġtop ics +T F +Ä ģ +br id +re ement +Ġunder lying +ĠM ember +Ġinvestig ating +Ġpregn ancy +Ġtouch down +ĠB and +ĠCall er +Ġinst ances +P P +w a +G ood +Ġ199 1 +ĠC old +Ġfear s +Ġrem arks +Ĩ Ĵ +at al +Ġm it +Ġexper iments +i pt +Col or +ind u +Up date +Ġ9 3 +A g +Ġ å +anc ouver +B oth +Ġjud ges +Ob ject +Ġst ere +umb n +Ġparticip ation +ĠSt ars +ĠJ ere +Ġweek ly +ĠB an +Ġconvers ations +ĠP itt +u z +ĠIndian a +ĠK ick +Ġinf ection +Ġhero es +Ġsett led +Ġstri p +Ġh al +Ġd ump +ĠS ci +Ġl es +Ġref erences +ĠU RL +ĠBr idge +Ġwant ing +For ce +Ġex clus +Me anwhile +m n +Ġg entle +m aker +sen al +ĠG ro +ou ri +ĠR ain +ĠAll iance +Ġl ift +el a +S D +ĠCle veland +Ġrank ed +Ġst adium +Ġdead ly +ä ¸ +Ġr iding +ar ia +ĠAr mor +Ġdocument ation +ĠGree ce +ree k +Ġl ens +ĠS a +Ġg ross +ĠE mer +ag ers +ĠD ub +ĠR h +ĠAM D +Ġarri val +Ġdes ert +Ġsupp lement +ĠRes p +Ġkn ee +Ġmarg in +f ont +og g +201 0 +ĠP ir +ĠP rom +iv als +Ġint ake +Ġdifferent ly +ug s +Ġb its +clud ed +Ġsearch ing +ĠD u +um ble +Ġfunction al +ĠBalt imore +ĠC ould +Ġdes ired +Ġcirc uit +ĠL yn +ĠG O +ĠF alse +re pre +' : +alt ies +Ġmin im +Ġdro ve +ĠSh ould +Ġh ip +Ġpro s +Ġut ility +ĠN ature +ĠM ode +P resident +o pp +r at +form ance +Ġconcent ration +Ġf ont +ĠB ud +Ġam id +Ġre vers +ĠM L +B ar +Ġinter action +Ġjur isd +Ġspell s +d ep +f il +Ġcivil ians +ut ter +ĠCo oper +ĠBel ow +Ġent rance +Ġcon vert +Ġcontrovers y +ow ered +Ġcontr ary +Ġar c +ĠExec utive +ĠOffic er +Ġpack ages +Ġprog ressive +w idth +Ġreserv ed +v ol +ĠSam sung +Ġprint ed +Ġcent ers +Ġintrodu ce +ĠKenn edy +Ġodd s +Ġsure ly +Ġindepend ence +Ġpass engers +repre ne +ĠBe h +Ġl oves +ĠESP N +Ġfac ilit +Ġident ical +Ġdo ct +Ġpartners hip +con f +ĠH ide +Ġconf used +ĠC ow +M en +Ġw rest +ĠIraq i +Ġh oles +ĠStud ies +Ġpregn ant +h ard +Ġsign als +I X +Ġpull ing +Ġgrad uate +Ġnomine e +D ate +Ġper mitted +Ġâ Ĥ¬ +ĠOk lahoma +St art +Ġauthor ized +Ġal arm +ĠC os +v an +Ġgener ations +c ular +Ġdr agon +ĠSoft ware +ĠEd ward +Ġcontro ller +S en +ge red +ĠV ik +Ġappro ached +Th ank +Ġcan ce +Ġform ula +ĠSm all +Ġweak ness +Ġr amp +it udes +j ud +Ġbrill iant +Ġacc us +s ource +Ġ8 00 +ĠE vil +S w +Ġhom eless +we ek +i ens +r ics +ĠTh ird +T O +Ġorgan ic +Ġpresent ation +ag h +ĠDown load +v ation +Ġas sembly +or able +hold ers +ĠBern ie +ĠHel p +Ġt ong +ĠF ight +Ġbe ach +B ook +ĠL ic +Ġr ush +ĠR ound +ou p +ĠMar x +Ġcalcul ated +ĠDe vil +ĠSar ah +Ġoccasion ally +Ġbul let +Av ailable +g ate +Ġ9 1 +Ġh osp +Ġprom ises +ĠH IV +ĠSt adium +ĠSt ock +ĠCorpor ation +g age +N G +ĠC redit +Ġs ne +ib l +Ġacc um +s uch +Ġterror ists +Ġconscious ness +ĠZ h +Ġdram a +ool a +pir ation +Ġlab our +ĠN in +Ġut ter +Ġdemocr atic +Ġass ass +il ation +Ġg est +Ġab road +Ġmet ab +Ġs orts +Ġfl av +U B +Ġm g +ĠNot hing +ĠO d +Ġmus ical +200 9 +Ġdro ps +oc ated +ater al +0000 00 +Ġg re +Ġequ ality +Ġburd en +Ġv ig +ĠLe ader +-------- ---- +Ġcere mony +Ġf ighter +Ġact ors +Ġ æ +am an +F i +Ġal ign +put er +Ġe lder +ĠN SA +Ġrepresent ation +ĠOnt ario +IT H +usal em +Ġharass ment +itz er +Ġsy mp +Ġbox es +ĠD R +Ġman ifest +at re +Ġ ^ +Ġd ies +le ton +Ġmiss ions +et he +Ġres olve +Ġfollow ers +Ġas c +Ġk m +l ord +am med +Ġsil ent +ĠAssoci ated +Ġtim ing +Ġprison ers +ĠK ings +ĠF ive +Ġtow er +Ġappro aches +Ġprecise ly +Ġb ureau +ĠM other +ĠI ss +Ġkey board +it ual +Ġfund ed +Ġstay ing +Ġpsych ological +Ġm ile +ĠLe on +ĠBar b +w ill +Ġw ider +ĠAtl antic +Ġt ill +ĠR ome +ro t +Ġaccomp an +Ġfl our +ac o +W orld +ĠExp ress +ĠY u +C or +Ġple ased +part y +Ġpoint ing +Ġinf lation +Ġro y +Ġ ), +ain er +Ġwedd ing +orm on +Ġrequ iring +Ġqual ified +Ġse gment +EN D +Ġs izes +e als +Ġcor rupt +ass ador +Ġcele b +Ġdream s +ĠM ess +Ġcheck ing +ĠV ersion +Ġprep aring +Ġact ively +ĠD iff +Ġl ux +ĠW inter +act eria +ĠN E +Ġdep uty +Ġtrans gender +Ġsum mary +Ġin her +er ies +ch ar +ĠY an +Ġkn ock +ĠP ath +Ġl ip +roll er +Ġimp ression +Ġcelebr ate +Ġsl ide +Ġgu ests +Ġcl ip +F S +Ġsav ings +Ġcapt ain +Ġleg acy +ĠDen ver +Ġw ounded +tab oola +AC T +Ġpurs ue +Ġo xy +Ġ q +Ġsem i +ĠN eed +ĠAff airs +Ġob sc +Ġcheck ed +Ġd ual +C ode +ĠM D +le m +ult y +Ġ © +ĠEl izabeth +Ġcent uries +ard ed +s rc +Ġev ident +enn is +at in +Ġunemploy ment +ĠMar io +Ġint im +Ch rist +Ġbi ological +Ġsold ier +ĠAdd ed +Ġm ath +ĠG il +Ġbi as +Ġd ating +ĠO cean +Ġm ice +M us +h ire +ĠT es +Ser ver +lim ited +S ize +Ġmet ers +Ġrock et +es see +Ġcertific ate +ĠIran ian +AS S +Ġgr id +D ec +Ġro lling +com mun +ĠSwed en +b ury +Ġtiss ue +Ġrac ism +ĠL ocal +Ġmyster y +Ġexam ine +Ġst em +Ġs its +Ġhop ed +ot ing +Ġdial ogue +Ġpers u +W atch +l ay +M AN +Ġch ronic +ĠPort land +mark et +ĠS EC +Ġparalle l +Ġsc andal +Ġcar ries +Ġphenomen on +h uman +ack er +ĠO x +Ġretire ment +tain ment +ov ie +ĠG ear +Ġd uties +Ġdo se +Ġsc roll +M B +in f +Ġsa uce +Ġland scape +red dit +ĠChampions hip +ĠRed dit +al id +Ġco in +Ġover s +Ġpost ing +ab out +Ġf el +and y +Ġb old +Ġfocus ing +e ffect +G R +Ġde emed +Ġrecommend ations +Ġste pped +Ġvot er +ĠDe ep +ĠInst agram +Ġmoder ate +ĠMary land +Ġrestrict ed +ĠM B +ĠCh all +Ġto b +Ġc ir +ĠO cc +ĠE ver +Ġcoll aps +IN FO += - +ĠP ict +ĠAcc ount +n c +Ġo ught +Ġex port +Ġdr unk +( ' +Ġw ise +ĠM ort +ne cess +Ġan cest +ĠInc re +Ġfrequ ent +m ir +Ġinterpret ation +Ġdepend ent +Ġco ins +ĠB ol +V ideo +ĠJust in +Ġfat al +Ġcook ing +Ġconf usion +ip her +Ġcust ody +ĠMor gan +om ach +ĠGovern or +Ġrestaur ants +el ing +Ġacknowled ged +Ġthe r +Ġgen es +ch ing +He y +Ġtact ics +ĠMex ican +Ġv end +Ġhe s +qu er +Ġnot ing +ĠCamer on +Ġtarget ing +ro ck +Ġcred its +Ġemot ions +Ġrepresent atives +new s +Ġlegisl ative +Ġrem oving +Ġtweet ed +ĠCar ter +ĠF ixed +Ġfor cing +Ġspeak er +Ġm ales +ĠViet nam +l ined +Ġconcept s +Ġvo ices +o ir +ĠT rib +W he +ĠJer usalem +ĠS ant +Ġc ul +Ġl ady +ĠHaw ai +Ġar ts +ĠIn n +ĠMach ine +ĠEm peror +Ġsl ot +g ly +ĠPro cess +II I +Ġathlet es +ĠTem ple +ĠRep resent +Ġpres c +Ġt ons +Ġgold en +Ġp unch +ĠG R +iver pool +Ġen act +Ġlob by +Ġm os +Ġpick ing +Ġlif etime +Ġcogn itive +E ach +z o +Ġd ub +Ġcons ists +ol n +Ġf estival +am ous +Ġint ellig +w ords +ĠSm art +Ġde le +Ġl apt +Ġmag ical +ĠS in +b us +ur ities +igh th +ĠRub y +ĠS ure +ol ving +Ġj un +O ST +Ġimp osed +Ġast ron +Ġcor rel +ĠN S +ĠK it +ĠF uture +b urn +Ġimm une +oc us +Ġcour ses +ĠSt ring +Ġle an +Ġg host +Ġout comes +Ġexp ense +Ġevery day +Ġaccept able +A h +Ġequ ipped +Ġor ange +F R +ĠD utch +Th ough +ĠR ank +Q U +ĠRober ts +wh at +re nd +Ġdisapp ear +Ġsp awn +ĠL am +o is +Ġdes erve +Ġmin imal +Ġnerv ous +ĠW ould +Ġro ok +ĠV ancouver +Ġres ign +sh ire +ĠW orks +ĠB uild +Ġafford able +ĠG ary +ĠAren a +Ġh anging +Ġimpl ications +ĠS ong +Ġmain taining +Ġgu ards +C ON +Ġder ived +Ġexecut ed +Ġthe ories +Ġqu oted +ĠAnd re +og a +sel ess +in fo +ĠBel g +Ġt ears +ĠSur v +Ġbirth day +ig ious +im mer +Ġspect rum +Ġarchitect ure +Ġrec ruit +arm a +T able +Ġmon sters +ĠG ov +Ġdest ination +Ġattract ive +Ġf oss +ĠMore over +Ġpres ents +TH E +Ġrep ly +pt on +Ġc um +Ġdel ight +Ġaffect s +Ġdon ations +ĠT oy +ĠH im +M ENT +Ġover come +it ched +ĠFant asy +ĠH at +ĠBe ast +b ott +Ġinvestig ations +R un +Ġhun ting +d i +f und +Ġs essions +est yle +Ġport ray +oid s +Y eah +Ġcommun icate +Ġcom edy +ĠY ang +Ġbel t +ĠMar ine +Ġpredict ed +Pl ay +Ġimportant ly +Ġremark able +Ġelim inate +D avid +Ġb ind +V ID +Ġadvoc ates +ĠG aza +im p +D B +ĠN a +ĠSim ilar +I ES +Ġchar ity +v as +m ath +Ġâ ĸ +ok er +nd um +Ġcap s +ĠH al +2 000 +e an +Ġfle et +Ġrec re +R ight +Ġsleep ing +ij ing +k ind +Ġdesign ated +à ¤ +Ġanim ation +ke e +ĠInt rodu +Ġ/ > +Ġdelay ed +Ġtrem end +Ġcur ious +U se +Ġle ct +d am +Ġinnov ation +ĠPoint s +Ġload ing +Ġdisp ute +ct ic +ird s +ĠB Y +Ġn urs +ĠVal ue +ION S +ĠH um +Ġtem plate +m ers +Ġappear ances +ĠEnter tainment +Ġtransl ation +Ġsa ke +Ġbene ath +Ġin hib +Ġe uro +abet es +Ġstud ying +ĠM as +Ġper ceived +Ġexam ined +Ġe ager +Ġco aches +Ġim per +ch i +Ġprodu ces +" ). +ĠEvery one +Ġm unicip +Ġg irlfriend +Ġh ire +ĠV ice +Ġsu itable +op y +Ġin equ +ĠD uke +f ish +f irst +ĠO bs +Ġinter ior +ĠBru ce +ĠR y +Ġanal ys +Ġconsider able +Ġfore cast +Ġf ert +ors hip +ĠD rug +ĠA LL +: " +th ur +ĠM ail +Ġball ot +Ġinst antly +ĠCh annel +Ġp icks +Ġ198 9 +Ġt ent +ol i +Ġcivil ian +b ling +ell o +b u +Ġin ch +Ġlog o +Ġcooper ation +Ġwal ks +Ġinvest ments +Ġimp rison +ĠF estival +ĠK y +Ġleg ally +Ġg ri +ch arg +S l +Ġthreat ening +du ction +fl ow +Ġdismiss ed +ibr aries +c ap +e le +ĠMc G +ĠHar vard +ĠConserv ative +ĠC BS +p ng +Ġro ots +ĠH aving +umb led +ĠF un +\ / +ĠS earch +ple x +Ġdiscuss ing +Ġcontin u +ĠT ai +ĠW ik +F ree +f it +Ġref use +Ġmanag ing +Ġsy nd +ip edia +w alk +Ġprofession als +Ġguid ance +Ġunivers ities +Ġas semb +unt u +F inally +AS E +ĠAut o +ĠH ad +Ġann iversary +L D +ĠD ur +ĠUlt imate +ih ad +pro duct +Ġtrans it +Ġrest ore +Ġexpl aining +Ġass et +Ġtransfer red +Ġbur st +ap olis +ĠMag azine +ĠC ra +ĠB R +gg ed +ĠH E +M ich +b et +ĠL ady +yl um +erv es +Ġme ets +wh ite +L og +Ġcorrespond ing +Ġins isted +G G +Ġsurround ed +Ġt ens +Ġl ane +Ġco inc +h ome +Ġexist ed +ect ed +ĠDou ble +lam m +Ġske pt +ex p +Ġper ception +ie v +ĠBe ing +o ft +Ġadop t +. : +] ; +Wind ows +Ġsatell ite +AS H +Ġinf ant +d escription +ĠMe anwhile +c m +oc a +ĠT reat +act or +Ġtob acco +ĠN orm +em ption +Ġfl esh +Ġj e +o op +ĠHe aven +Ġbe ating +an im +Ġgather ing +Ġcult iv +G O +ab e +ĠJon athan +ĠSaf ety +Ġbad ly +pro t +Ġcho osing +Ġcontact ed +Ġqu it +Ġdist ur +Ġst ir +Ġto ken +D et +ĠP a +Ġfunction ality +00 3 +s ome +Ġlimit ations +Ġmet h +b uild +con fig +N T +re ll +ble m +ĠM om +Ġveter ans +ĠH u +Ġtrend s +are r +ĠG iven +ĠCa ption +m ay +AS T +Ġwond ering +ĠCl ark +n ormal +Ġsepar ated +Ġdes p +st ic +b rew +Ġrel ating +ĠN ik +ĠF arm +Ġenthus i +g ood +d eb +Ġactiv ist +Ġm art +Ġexplos ion +ĠEconom ic +L ink +Ġins ight +Ġconven ient +Ġcounter part +su pport +ĠV irt +ag en +ĠTenn essee +ĠSim on +ĠA ward +OC K +ĠF igure +Ġoverse as +Ġpr ide +ĠC as +n ote +m g +C urrent +Ġdispl ays +cont ent +Ġtravel ing +Ġhosp itals +ĠFin ancial +ĠP ast +Ġdefend ant +Ġstream ing +m ble +ĠBer lin +uk i +Ġdist ribut +Ġant ib +Ġch ocolate +ĠCast le +Ġinter rupt +ĠR ow +Ġconvers ion +Ġbug s +ĠR ather +li est +L Y +ĠJe an +com mon +ak h +Ġ1 30 +ot ton +ĠDe an +Ġam endment +Ġgame play +ĠWar ren +od a +Ġhigh lights +Ġir re +ĠNAT O +Ġball s +Ġdemand ing +U RE +ĠL uke +F igure +st op +on ia +z one +iz ers +ĠW R +Ġaward ed +Ġregul atory +ĠH art +ĠS N +pl ing +Ġs our +ĠP ixel +us ive +Ġf et +ĠS ent +Ġautom atic +Ġf er +vern ment +ĠKh an +T ON +f ather +Ġextraord inary +th rop +ĠP ython +ĠG PU +Ġsex ually +Ġdesk top +it ivity +ĠAnton io +Ġo rient +Ġe ars +ob by +ous es +vertis ements +Ġmanufacture rs +ic ient +min ute +Ġconv iction +Ġg arden +p ublic +Ġsatisf ied +f old +O K +Ġin hab +ĠTh ink +Ġprogram me +Ġst omach +Ġcoord in +Ġh oly +Ġth reshold +Ġr het +Ġser ial +Ġemploy ers +ĠEvery thing +ra h +Ġb other +Ġbr ands +Val ue +ĠT ed +ĠPlan et +Ġp ink +ĠFurther more +s a +P E +re ck +ĠUS D +ot te +Ġ& & +Ġland ed +g ets +Ġprodu cers +Ġhealth care +Ġdomin ant +Ġdest ro +Ġam ended +ch ron +Ġf its +ĠSy d +ĠAuthor ity +AT CH +Ġfight s +ĠL LC +Ġ-- - +ĠCor p +Ġtox ic +spe cific +ĠC orn +ĠChe l +Ġtele phone +ĠP ant +Ġmyster ious +aun ch +od ox +med ia +Ġwitness es +ag u +Ġquestion ed +ĠBre xit +ĠRem ember +ene z +Ġend orse +iat ric +ĠId ent +Ġridic ulous +1 10 +Ġpr ayer +Ġscient ist +Ġ19 50 +ĠA qu +Ġunder ground +ĠU FC +m are +ĠL ater +w ich +Ġsubsc rib +Ġhost s +Ġer r +Ġgr ants +ant om +Ġsum mon +ear ly +ĠC lear +ĠPr im +Ġsusp ension +Ġguarant eed +app er +Ġr ice +ĠSe an +ĠSh in +Ġrefere ndum +Ġfl ed +r ust +Ġ3 60 +ter y +Ġsh ocked +B R +ĠO il +ĠAll ah +Ġpart ly +Ġign or +Ġtrans mission +Ġhom osexual +ivers al +Ġhop efully +ãĤ ¤ +Ġless on +L eg +Ġ .. +Y et +t able +app ropri +re tt +Ġbo ards +Ġincor rect +Ġb acteria +ar u +am ac +Ġsn ap +.' " +Ġpar ad +t em +he art +Ġav ailability +Ġw isdom +Ġ( + +Ġpri est +ĠÂł ĠÂł +O pen +Ġsp an +Ġparam eter +Ġconv ince +Ġ( %) +r ac +Ġf o +Ġsafe ly +Ġconver ted +ĠOlymp ic +Ġres erve +Ġhe aling +ĠM ine +M ax +Ġin herent +ĠGra ham +Ġinteg rated +D em +Ġpip eline +Ġapp lying +Ġem bed +ĠCharl ie +Ġc ave +200 8 +Ġcons ensus +Ġre wards +P al +ĠHT ML +Ġpopular ity +look ing +ĠSw ord +ĠAr ts +' ) +Ġelect ron +clus ions +Ġinteg rity +Ġexclus ively +Ġgr ace +Ġtort ure +Ġburn ed +tw o +Ġ18 0 +P rodu +Ġent reprene +raph ics +Ġg ym +ric ane +ĠT am +Ġadministr ative +Ġmanufacture r +Ġ vel +ĠN i +Ġisol ated +ĠMedic ine +Ġback up +Ġpromot ing +Ġcommand er +Ġfle e +ĠRus sell +Ġforg otten +ĠMiss ouri +Ġres idence +m ons +Ġrese mb +Ġw and +Ġmeaning ful +P T +Ġb ol +Ġhe lic +Ġwealth y +Ġr ifle +str ong +row ing +pl an +as ury +â̦ . +Ġexpand ing +ĠHam ilton +Ġrece ives +S I +eat ures +ĠAn im +RE E +P ut +Ġbrief ly +ri ve +Ġstim ul +Ġ`` ( +Ġ __ +Ġch ip +Ġha z +Ġpri ze +ĠTh ings +AC E +ul in +d ict +ok u +Ġassoci ate +ock ets +y outube +St ory +ateg ory +Ġm ild +ail ing +ĠY e +O rig +ĠK a +or ig +Ġpropag anda +Ġan onymous +Ġstrugg led +Ġout rage +AT ED +ĠBe ijing +r ary +Ġle ather +Ġworld s +Ġbroad er +12 5 +id al +ĠBet ter +Ġt ear +E xt +Ġpropos als +Ġit er +ĠSqu ad +Ġvol unt +m i +D id +ĠP u +p in +Ġspeak ers +Ġb orders +Ġfig ured += ' +Ġsimultane ously +aed a +Ġcharg ing +Ġur ged +Ġcon j +25 6 +ĠG ordon +mer ce +Ġdocument ary +Sh are +it ol +ON E +ĠG arden +h att +ĠThom pson +ane ous +ap ore +Ġt anks +Ġless ons +tr ack +Ġout standing +Ġvolunte ers +Ġsp ray +Ġmanag ers +l arge +Ġcamp s +Ġart ificial +ĠR u +Ġb ags +th al +Ġcompat ible +ĠBl ade +Ġf ed +Ġarg ues +F I +Ġunf air +Ġcor n +Ġoff set +Ġdirect ions +Ġdisappoint ed +ĠCon vention +Ġview ing +M E +oc ity +Ġtown s +Ġlay ers +Ġro lled +Ġjump ed +Ġatt ribute +Ġun necess +inc oln +Ġsupp ose +ĠNet her +ch a +Ġbur ied +Ġsix th +B en +ress ing +OU R +Ġw ound +Ġcy cl +Ġmechan isms +Ġcongress ional +ĠE lement +Ġagre ements +Ġdec or +Ġclos est +ĠM it +Go ogle +} } +Ġm ixture +Ġflu id +S ign +ĠSch olar +Ġp ist +ask et +ab ling +Ġrac ing +he ro +ri el +ass y +Ġche aper +b en +Ġvert ical +amac are +ĠRead ing +g ments +Ġhelic op +Ġsacr ifice +ay a +p aren +V A +ĠL es +ĠStud io +Ġviol ations +ĠAn na +ac er +é ¾ +ĠR at +ĠBe ck +ĠD ick +ĠA CT +Ġcomp osition +Ġtext ure +ĠO wn +Ġsmart phone +ĠN A +Ġfor b +im port +Ġdef ending +il st +re r +Ġo h +ĠJere my +Ġbank ing +cept ions +Ġrespect ive +/ . +Ġdr inks +ĠW i +Ġb ands +ĠL iverpool +Ġg rip +ĠB uy +Ġopen ly +Ġreview ed +per t +Ġver ify +ĠCo le +ĠW ales +M O +Ġun pre +Ġshel ter +ĠIm perial +Ġgu i +ĠD ak +Ġsuggest ions +Ġexplicit ly +Ġsl ave +Ġblock chain +Ġcompet ing +Ġprom ising +S ON +Ġsoc cer +Ġconst itution +4 29 +Ġdist ract +ĠU ser +es ides +ĠMet hod +ĠTok yo +Ġaccompan ied +Cl ient +s ur +al og +Ġident ification +Ġinv asion +as ma +Ġindust ries +pp ers +Ġsub tle +ĠUn it +n atural +Ġsurv ived +Ġfl aw +ĺ ħ +ĠH oll +Ġdef icit +Ġtut orial +ĠCh ance +Ġarg uing +Ġcontem porary +Ġinteg ration +for ward +Ġt um +it is +Ġh iding +ĠD omin +ĠT an +ĠB uilding +ĠV in +Ġspokes person +ĠNot es +Ġemer ging +Ġprepar ation +Ġpro st +Ġsuspect s +Ġaut onom +D escription +Ġdeal t +ĠP ear +Ġstead y +Ġdecre ased +Ġso vere +ĠCl in +Ġgrad ually +ors es +ĠW AR +S erv +ãĤ ¢ +h r +Ġd irty +ĠB arn +ĠB C +Ġd il +Ġcal endar +Ġcompl iance +Ġch amber +b b +Ġpass enger +ate ful +ĠT itle +ĠSyd ney +ĠG ot +Ġdark ness +Ġdef ect +Ġpack ed +ass ion +Ġgod s +Ġh arsh +IC K +le ans +Ġalgorith m +Ġoxy gen +Ġvis its +Ġbl ade +Ġkil omet +ĠKent ucky +Ġkill er +P ack +enn y +Ġdiv ine +Ġnom ination +be ing +Ġeng ines +Ġc ats +Ġbuff er +ĠPh ill +Ġtra ff +AG E +Ġtong ue +Ġrad iation +ere r +m em +ĠExpl icit +é¾ į +Ġcou ples +Ġphys ics +ĠMc K +Ġpolit ically +aw ks +ĠBl oom +Ġwor ship +e ger +ut er +ĠF O +Ġmat hemat +Ġsent enced +Ġdis k +ĠM arg +Ġ/ * +P I +Ġoption al +Ġbab ies +Ġse eds +ĠScott ish +Ġth y +] ] +ĠHit ler +P H +ng th +Ġrec overed +ing e +Ġpow der +Ġl ips +Ġdesign er +Ġdis orders +Ġcour age +Ġch aos +" },{" +Ġcar rier +b ably +H igh +ĠR T +es ity +l en +Ġrout es +u ating +F il +N OT +w all +s burgh +Ġeng aging +ĠJava Script +ore r +li hood +Ġun ions +ĠF ederation +ĠTes la +Ġcomple tion +ĠT a +Ġprivile ge +ĠOr ange +Ġne ur +paren cy +Ġb ones +Ġtit led +Ġprosecut ors +ĠM E +Ġengine er +ĠUn iverse +ĠH ig +n ie +o ard +Ġheart s +ĠG re +uss ion +Ġmin istry +Ġpen et +ĠN ut +ĠO w +ĠX P +in stein +Ġbul k +S ystem +ic ism +ĠMarket able +Ġpre val +Ġpost er +Ġatt ending +ur able +Ġlicens ed +ĠG h +et ry +ĠTrad able +Ġbl ast +à ¤ +ĠTit an +ell ed +d ie +H ave +ĠFl ame +Ġprof ound +Ġparticip ating +Ġan ime +ĠE ss +Ġspec ify +Ġregard ed +ĠSpe ll +Ġs ons +own ed +Ġm erc +Ġexper imental +land o +h s +ĠDun geon +in os +Ġcomp ly +ĠSystem s +ar th +Ġse ized +l ocal +ĠGirl s +ud o +on ed +ĠF le +Ġconstruct ed +Ġhost ed +Ġsc ared +act ic +ĠIs lands +ĠM ORE +Ġbl ess +Ġblock ing +Ġch ips +Ġev ac +P s +Ġcorpor ation +Ġo x +Ġlight ing +Ġneighb ors +ĠU b +ar o +Ġbe ef +ĠU ber +F acebook +ar med +it ate +ĠR ating +ĠQu ick +Ġoccup ied +Ġaim s +ĠAdd itionally +ĠInt erest +Ġdram atically +Ġhe al +Ġpain ting +Ġengine ers +M M +ĠM ust +Ġquant ity +P aul +Ġearn ings +ĠPost s +st ra +ãĥ¼ ãĥ +Ġst ance +Ġdro pping +sc ript +Ġd ressed +M ake +Ġjust ify +ĠL td +Ġprompt ed +Ġscr ut +Ġspeed s +ĠGi ants +om er +ĠEd itor +Ġdescrib ing +ĠL ie +ment ed +Ġnow here +oc aly +Ġinst ruction +fort able +Ġent ities +Ġc m +ĠN atural +Ġinqu iry +Ġpress ed +iz ont +for ced +Ġra ises +ĠNet flix +ĠS ide +Ġout er +Ġamong st +im s +ows ki +Ġclim b +ne ver +Ġcomb ine +d ing +Ġcomp r +Ġsignific ance +Ġremem bered +ĠNev ada +ĠT el +ĠSc ar +ĠWar riors +ĠJ ane +Ġcou p +b as +Ġtermin al +, - +O H +Ġt ension +Ġw ings +ĠMy ster +�� �� +ĠUn like +val id +viron ments +ĠAl i +Ġn aked +book s +ĠM un +ĠG ulf +Ġd ensity +Ġdim in +Ġdesper ate +Ġpres idency +Ġ198 6 +h y +IN D +Ġun lock +im ens +Ġhand led +ĠE b +Ġdisapp eared +Ġgen re +Ġ198 8 +Ġdetermin ation +St ream +ik o +ap ters +Ġacknow ledge +J an +Ġcapital ism +P at +Ġ20 20 +Ġpain ful +Ġcur ve +Ġbom bs +st orm +ĠMet al +en cer +ĠF ig +ĠA aron +anc hes +Ġins piration +Ġexha ust +t ains +ash i +Ġdesc ript +Ġr itual +ĠChel sea +Ġpromot ion +ĠH ung +ĠW ard +iv a +ĠE T +Ġto ss +all ow +ĠFranc is +D ep +Ġhapp iness +ĠGl ass +Ġbet a +Ġstreng then +N E +o a +Ġbutt ons +ĠMur ray +Ġkick ed +Qu est +ĠT alk +ĠS everal +ĠZ ero +Ġdr one +ul k +Ġc am +ĠM obile +Ġprevent ing +Ġret ro +ĠA x +Ġcru el +Ġflo at +. ), +Ġfil ing +ĠGr ant +ĠB or +Ġr ib +Ġchampions hip +ĠM erc +Ġsty les +Ġc ake +Ġbuild s +ĠS elf +io x +Ġep ic +oy d +B el +ĠSt ew +. ( +ah u +ĠBe yond +Ġout s +Ġsol o +ĠT ree +Ġpres erve +Ġt ub +AR E +ro c +ĠIm pro +ĠW right +Ġbu nd +Ġtr aged +Ġoccas ional +b ian +Sec ond +r ons +Ġinter actions +form ed +s ing +Ġown s +Ġh ockey +Gener al +Ġlog ical +Ġexp end +Ġesc al +ĠGr iff +ĠC rown +ĠRes erve +Ġsto pping +Ġexc use +sec ond +Ġoper ated +Ġre aches +ĠMal ays +Ġpoll ution +ĠBrook lyn +Ġde lete +Ġhas h +Bl ock +ah a +âĢ ³ +Ġsh orter +p iece +> >> +ĠM ormon +t or +Ġpartic les +ĠB art +ry ption +Ġad min +Ġsqu ee +VID IA +Ġcreat or +iam eter +ic ular +N BC +Ġgrab bed +Ġn odd +Ġr ated +Ġrot ation +Ġgr asp +Ġexcess ive +ĠE C +ĠWh it +Ġinvent ory +ault s +ĠF B +Ġe cosystem +Ġbill ions +Ġvent ure +n amed +Ġdef ender +out e +Inst ead +ir able +W ar +Ġassum ption +Ġb ite +Ġearth qu +t ail +sp ace +Ġgif ts +boy s +Ġinev itable +Ġstruct ural +Ġbenef icial +Ġcompe lling +h ole +erv ation +Ġco at +o j +inc arn +ĠY ears +Ġdetermin ing +Ġrhet oric +Ġbound aries +Ġwh ites +A nt +add y +) - +ra ham +eter min +Ġhar vest +ĠCon c +Ġlapt op +ĠM atch +Ġenjoy ing +cc a +oll ar +Ġtri ps +Ġadd iction +ĠS ak +Ġpow ered +Ġc ous +ĠRuss ians +ie re +Ġret rie +qu ality +Ġdiff er +Ġking dom +ĠL aur +ĠCap itol +Ġcon clusions +ĠAl tern +ĠN av +Ġtrans parent +B ER +G roup +ĠCom plete +Ġinf er +Ġint rig +Ġins ane +R O +oph ob +is en +qu al +Mich ael +Ġm useum +ĠP ope +Ġres et +r ative +f ive +Ġagg reg +itte es +osit ory +Ġcar b +ĠRec ord +Ġdec ides +ĠF ix +Ġexcept ions +ĠCommission er +un s +ĠEnvironment al +Ġlegend ary +ist ence +Ġtun nel +k m +Ġins ult +Ġt roll +Ġsh ake +Ġdet ention +qu es +ĠCh rome +ĠF iles +Ġsub t +Ġprospect s +Ġpro l +re nder +pro of +Ġperform ances +St r +Ġh ref +ern ame +Ġachieve ment +Ġf ut +F ull +ĠLe ban +go ogle +ãĥ Ī +amp a +May be +Ġproject ed +ĠE mb +Ġcol leg +Ġa wards +Ġâ Ķ +G old +ĠBl ake +ĠR aj +if ting +Ġp ending +Ġinst inct +Ġdevelop ments +Con nect +ĠM and +ĠW ITH +ĠPhilipp ines +prof ile +Ġalt ogether +ĠB und +ĠT D +oo oo +amp ed +ip h +Ġste am +Ġold est +Ġdet ection +ul pt +Ġ ç +ĠWay ne +200 6 +f a +Ġcir cles +ĠF u +Ġdon ors +appropri ate +ĠDak ota +j amin +Ġmotiv ated +Ġpurch ases +ĠLouis iana +ĠS pl +Ġgl obe +Ġ10 5 +z ip +c all +Ġdepart ments +Ġsustain able +10 5 +ĠO P +if iers +Ġprevent ed +Ġinc omp +ĠComm ander +Ġdom inated +Ġ » +Ġinvest ed +Ġcomplex ity +Ġin cl +Ġens uring +Ġreal m +yn c +ĠInd ependent +r ained +ĠJ en +ĠFl ight +Ġat he +Ġspec ulation +ĠT E +oc ate +t ic +Ġpl aint +her ry +Ġto y +Ġ1 11 +Ġpl ates +st atus +ĠIs a +Ġdev oted +C op +ĠE S +25 5 +ur rency +M ain +Ġsl aves +Ġpe pper +Ġqu otes +Ġce iling +ĠF ish +Ġtrans formation +Ġfra ction +Ġadvant ages +Ġto ile +Ġstun ning +Ġmo ist +bre aking +s i +ĠL ocation +ĠMed ium +Ġtext s +Ġu gly +Ġb io +. âĢĶ +ĠB ased +Ġtr ains +ĠW ing +ĠAn cient +ĠRec ords +ĠH ope +Spe cial +ades h +ob i +[ / +Ġtempor arily +V er +h u +os er +Ġover night +Ġm amm +ĠTre asury +ĠV enezuel +ĠMeg a +Ġt ar +Ġexpect s +bl ack +or ph +\\ \\ +Ġaccept ance +Ġrad ar +s is +Ġjun ior +Ġfram es +Ġobserv ation +ac ies +P ower +ĠAdv anced +M ag +olog ically +ĠMe chan +Ġsent ences +Ġanaly sts +augh ters +force ment +Ġv ague +Ġcl ause +Ġdirect ors +Ġeval uate +Ġcabin et +M att +ĠClass ic +A ng +Ġcl er +ĠB uck +Ġresear cher +Ġ16 0 +Ġpoor ly +Ġexperien cing +ĠP ed +ĠMan hattan +Ġfre ed +Ġthem es +ad vant +Ġn in +Ġpra ise +10 4 +ĠLib ya +b est +Ġtrust ed +Ġce ase +Ġd ign +D irect +Ġbomb ing +Ġm igration +ĠSci ences +Ġmunicip al +ĠA verage +Ġgl ory +Ġreve aling +Ġare na +Ġuncertain ty +Ġbattle field +ia o +G od +Ġc inem +ra pe +el le +ap ons +Ġlist ing +Ġwa ited +Ġsp otted +ke ley +ĠAud io +e or +ard ing +idd ing +ig ma +ĠN eg +Ġl one +Ġ ---- +ex e +d eg +Ġtrans f +Ġwas h +Ġsl avery +Ġexpl oring +ĠW W +ats on +Ġen cl +l ies +ĠC reek +Ġwood en +Man ager +ĠBr and +um my +ĠAr thur +Ġbureau cr +Ġbl end +ar ians +F urther +Ġsupposed ly +Ġwind s +Ġ19 79 +Ġgrav ity +Ġanalys es +ĠTra vel +ĠV eter +Ġd umb +Ġaltern ate +g al +Ġconsum ed +Ġeffect iveness +.' ' +Ġpath s +ond a +L A +ĠStr ong +Ġen ables +Ġesc aped +Ġ" " +Ġ1 12 +Ġ198 3 +Ġsm iled +Ġtend ency +F ire +Ġp ars +ĠR oc +Ġl ake +Ġf itness +ĠA th +ĠH orn +Ġh ier +Ġimp ose +m other +Ġp ension +ic ut +bor ne +ic iary +. _ +ĠS U +Ġpol ar +is y +eng u +itial ized +AT A +w rite +Ġexerc ises +ĠD iamond +ot ypes +Ġharm ful +on z +Ġprint ing +st ory +Ġexpert ise +ĠG er +Ġtraged y +ĠF ly +Ġd ivid +amp ire +st ock +M em +Ġre ign +Ġun ve +Ġam end +ĠProp het +Ġmut ual +ĠF ac +Ġrepl acing +H ar +ĠCirc uit +Ġthro at +ĠSh ot +Ġbatter ies +Ġto ll +Ġaddress ing +ĠMedic aid +Ġp upp +ĠN ar +ol k +Ġequ ity +M R +ĠHis pan +ĠL arge +m id +D ev +Ġexp ed +Ġdem o +ĠMarsh all +erg us +Ġf iber +Ġdiv orce +ĠCre ate +Ġsl ower +ĠPark er +ĠStud ent +ĠTr aining +Ret urn +ĠT ru +Ġc ub +ĠRe ached +Ġpan ic +Ġqu arters +Ġre ct +Ġtreat ing +Ġr ats +ĠChristian ity +ol er +Ġsac red +Ġdecl are +ul ative +et ing +Ġdeliver ing +est one +Ġt el +ĠL arry +Ġmet a +ac cept +art z +ĠRog er +hand ed +Ġhead er +Ġtra pped +ĠCent ury +Ġkn ocked +ĠOx ford +Ġsurviv ors +b ot +Ġdemon stration +Ġd irt +Ġass ists +OM E +ĠD raft +ortun ate +fol io +pe red +ust ers +g t +ĠL ock +Ġjud icial +ver ted +Ġsec ured +out ing +ĠBook s +Ġhost ing +Ġlif ted +l ength +Ġj er +Ġwhe els +ĠR ange +umbn ails +Ġdiagn osis +te ch +ĠStew art +ĠP ract +Ġnation wide +Ġde ar +Ġoblig ations +Ġgrow s +Ġmand atory +Ġsusp icious +! ' +A pr +G reat +Ġmort gage +Ġprosecut or +Ġeditor ial +ĠK r +Ġprocess ed +ung le +Ġflex ibility +Ear lier +ĠC art +ĠS ug +Ġfoc uses +Ġstart up +Ġbre ach +ĠT ob +cy cle +ãĢ Į +ro se +Ġb izarre +ãĢ į +Ġveget ables +$ $ +Ġret reat +osh i +ĠSh op +ĠG round +ĠSt op +ĠHawai i +ĠA y +Per haps +ĠBe aut +uff er +enn a +Ġproduct ivity +F ixed +cont rol +Ġabs ent +ĠCamp aign +G reen +Ġident ifying +Ġreg ret +Ġpromot ed +ĠSe ven +Ġer u +ne ath +aug hed +ĠP in +ĠL iving +C ost +om atic +me ga +ĠN ig +oc y +Ġin box +Ġem pire +Ġhor izont +Ġbr anches +Ġmet aph +Act ive +ed i +ĠFil m +ĠS omething +Ġmod s +inc ial +ĠOrig inal +G en +Ġspir its +Ġear ning +H ist +Ġr iders +Ġsacr ific +M T +ĠV A +ĠS alt +Ġoccup ation +ĠM i +Ġdis g +lic t +Ġn it +Ġn odes +e em +ĠP ier +Ġhat red +ps y +ãĥ ī +Ġthe ater +Ġsophistic ated +Ġdef ended +Ġbes ides +Ġthorough ly +ĠMedic are +Ġbl amed +arent ly +Ġcry ing +F OR +pri v +Ġsing ing +ĠI l +Ġc ute +o ided +olit ical +ĠNe uro +å ¤ +Ġdon ation +ĠEag les +ĠG ive +T om +Ġsubstant ially +ĠLic ense +ĠJ a +Ġg rey +ĠAn imal +ĠE R +ĠU nd +Ġke en +Ġconclud e +ĠMississ ippi +Eng ine +ĠStud ios +P ress +o vers +ll ers +Ġ3 50 +ĠR angers +Ġr ou +ert o +E p +iss a +iv an +Ġse al +ĠReg ist +dis play +Ġwe aken +u um +ĠComm ons +ĠS ay +Ġcult ures +Ġl aughed +Ġsl ip +Ġtreat ments +iz able +m art +ĠR ice +Ġbe ast +Ġob esity +ĠLa ure +ig a +Wh ich +hold er +Ġelder ly +Ġp ays +Ġcompl ained +Ġc rop +Ġpro c +Ġexplos ive +ĠF an +ĠAr senal +A uthor +ef ul +Ġme als +Ġ( - +id ays +Ġimag ination +Ġann ually +Ġm s +as ures +H ead +ik h +m atic +Ġboy friend +ĠCom puter +Ġb ump +Ġsur ge +ĠCra ig +ĠKir k +D el +medi ate +Ġscen arios +ĠM ut +ĠSt ream +Ġcompet itors +Ù Ħ +ĠStan ford +ĠRes ources +az ed +b age +Ġorgan is +ĠRe lease +Ġsepar ately +Ġha bits +Ġmeasure ments +ĠCl ose +Ġaccomp any +Ġg ly +Ġt ang +ĠR ou +Ġplug in +Ġcon vey +ĠChall enge +oot s +j an +Ġcur s +ĠRel ations +ke eper +Ġapproach ing +p ing +Spe aking +Ġarrang ement +ĠV I +are ttes +Ġaffect ing +Ġperm its +b ecause +Ġu seless +ĠH us +!! !! +Ġdestro ying +Un fortunately +Ġfasc inating +S em +Ġelect oral +Ġtrans parency +ĠCh aos +Ġvolunte er +Ġstatist ical +Ġactiv ated +ro x +We b +H E +ĠHamp shire +is ive +M ap +Ġtr ash +ĠLaw rence +st ick +C r +Ġr ings +EX T +Ġoper ational +op es +D oes +ĠEv ans +Ġwitness ed +P ort +Ġlaunch ing +ec onom +w ear +ĠPart icip +um m +cul es +ĠR AM +ĠT un +Ġass ured +Ġb inary +Ġbet ray +Ġexpl oration +ĠF el +Ġad mission +it ated +S y +Ġav oided +ĠSim ulator +Ġcelebr ated +ĠElect ric +¥ ŀ +Ġcl uster +itzer land +he alth +L ine +ĠN ash +at on +Ġsp are +Ġenter prise +ĠD IS +clud es +Ġfl ights +Ġreg ards +ĠÃ Ĺ +h alf +Ġtr ucks +Ġcontact s +Ġunc ons +ĠCl imate +Ġimm ense +N EW +oc c +ect ive +Ġemb od +Ġpat rol +Ġbes ide +Ġv iable +Ġcre ep +Ġtrig gered +ver ning +Ġcompar able +q l +Ġg aining +ass es +Ġ( ); +ĠG rey +ĠM LS +s ized +Ġpros per +" ? +Ġpoll ing +Ġsh ar +ĠR C +Ġfire arm +or ient +Ġf ence +Ġvari ations +g iving +ĠP i +osp el +Ġpled ge +Ġc ure +Ġsp y +Ġviol ated +Ġr ushed +Ġstro ke +ĠBl og +sel s +ĠE c +,' ' +Ġp ale +ĠColl ins +ter ror +ĠCanad ians +Ġt une +Ġlabor atory +Ġn ons +t arian +Ġdis ability +ĠG am +Ġsing er +al g +ĠSen ior +Ġtrad ed +ĠWar rior +Ġinf ring +ĠFrank lin +Ġstr ain +ĠSwed ish +Ġsevent h +ĠB enn +ĠT ell +Ġsynd rome +Ġwond ered +id en +++ ++ +ig o +Ġpur ple +Ġjournal ism +Ġreb el +Ġf u +bl og +Ġinv ite +ren cies +ĠCont act +Is rael +ĠCont ent +Ġche er +Ġbed room +ĠEngine ering +ĠQue ens +Ġd well +ĠPlay Station +ĠD im +ĠCol on +l r +Ġoper ates +Ġmotiv ation +US A +ast ered +C ore +ĠTr uth +ol o +OS E +ĠMem ory +Ġpred ec +Ġan arch +Ġ19 20 +ĠY am +à ¨ +b id +Ġgr ateful +Ġexc itement +Ġtre asure +Ġlong est +ct ive +Ġdes erves +Ġreserv es +Ġcop s +ĠOtt awa +ĠEgypt ian +ank ed +Ġart if +Ġhypot hesis +: / +Ġpurch asing +Ġlove ly +H P +Ġdiv ide +Ġstrict ly +Ġquestion ing +Ġtaxp ayers +ĠJ oy +Ġroll s +ĠHe avy +Ġp orts +Ġmag netic +Ġinf lamm +Ġbr ush +t ics +â ĪĴ +Ġbott les +pp y +Ġp add +ãĤ ¯ +m illion +Ġdevast ating +Ġcomp iled +Ġmed ication +Ġtw elve +ĠPer ry +Sp ace +im b +y our +Ġle aked +ĠT ar +Ġun ity +Ġinfect ed +Ġtravel ed +ID E +ĠMc Donald +t xt +ĠPr inc +Ġinter ven +ĠTai wan +ĠP ow +Ġbe aring +ĠTh read +Ġz ones +iz ards +un ks +Ch apter +ll or +Ġ · +Ġw ounds +Ġdisc retion +Ġsucceed ed +ik ing +Ġicon ic +C all +Ġscreen ing +ĠM is +ict s +Ġmin isters +Ġsepar ation +Pl ayer +Ġb ip +Ġbel oved +Ġcount ing +ĠE ye +ar ound +ing ing +Ġtable t +Ġoff ence +in ance +h ave +ĠInf o +ĠNin ja +Ġprotect ive +ĠC ass +M ac +ĠQual ity +N orth +Ġ ic +ĠCub a +ĠChron icle +ĠPro perty +Ġfast est +ot os +ĠG erm +OW N +Ġbo om +ĠStan ley +ergus on +Ġcle ver +Ġent ers +m ode +ter ior +ĠS ens +Ġlin ear +AR K +Ġcomp aring +Ġpure ly +Ġsaf er +ĠPot ter +Ġc ups +R T +Ġgl uc +Ġatt ributed +Ġdu pl +ĠP ap +Ġprec ious +Ġp a +iction ary +ĠT ig +ĠTo o +ol utions +st an +Ġrob ots +Ġlob b +Ġstat ute +Ġprevent ion +w estern +16 0 +ĠAct ive +ĠMar ia +h al +N one +ell ar +ĠK B +ĠPart ners +ĠSing le +ĠFollow ing +ang o +ac ious +Ġth ou +Ġk g +Ġinflu ential +ĠFriend s +S ur +ain ted +Ġfor ums +Ġst arter +Ġcitizens hip +ĠE lection +on ge +ot ation +os ph +;; ;; +ut ical +p ur +ere n +Ġaccus ations +bit ious +ab bit +ĠOr d +Post ed +ir k +Ġsens itivity +ic he +ĠAm y +ĠF ab +Ġsum mit +Ġped est +Ġrub ber +Ġagric ultural +Ġcan cel +A E +Ġin aug +Ġcont am +Ġfirm ly +i w +st age +ĠK an +Ġt ier +Ġinv ention +Ġtransl ated +ĠR ules +B ox +Tw itter +ID S +Ġp izza +Ġdeb ug +ĠD rop +v s +Ġh orses +b ig +Ġb oring +Ġh ood +ĠMcC ain +at ched +ĠBro s +Ġsk ip +Ġess ay +st at +ĠLeg ends +Ġam munition +au c +Ġshoot er +Ġun h +Ġsuppl ied +Ġgener ic +ĠS K +ib an +yr ics +Ġ25 5 +Ġclim bing +Form er +Ġfl ip +Ġjump ing +Ġfrust ration +ĠTer ry +Ġneighborhood s +Ġmed ian +be an +Ġbr ains +Follow ing +Ġsh aped +Ġdraw s +Ġal tered +J ack +Ġrecip es +Ġsk illed +we alth +ach i +e lection +Ġbehavi ors +de als +ĠU ntil +F e +Ġdecl aration +mar ks +ĠBet ween +cel ona +Ġres on +Ġbub ble +Am ong +Ġim perial +G S +Ġfemin ist +200 5 +ĠK yle +Ġaccount ing +ĠTe le +ĠT yr +Ġconnect ing +Ġre hab +ĠP red +s im +Ġmeant ime +Ġphys ician +M W +ĠCamp bell +ĠBr andon +Ġcontribut ing +ĠR ule +ĠWe ight +ĠN ap +Ġinter active +Ġv ag +Ġhel met +ĠCom b +f our +Ġsh ipped +Ġcomple ting +ĠP D +PD ATE +Ġspread ing +Ġsc ary +erv ing +ĠG as +Ġfr ank +s chool +Ġrom antic +Ġstab il +R ob +Ġaccur ately +Ġac ute +ĠH ann +Ġsymbol s +Ġcivil ization +ĠA W +Ġlight ning +Ġcons iders +Ġven ue +Ġ × +Ġo ven +ĠS F +h is +Ġn u +ĠLear n +Ġpe oples +Ġst d +Ġsle e +Ġs lic +ĠStat istics +Ġcor ners +ĠB aker +Ġ: ) +ment ation +ol ver +Ġlaugh ing +ĠT odd +ond e +ĠH ills +Ġn uts +ĠW oman +pl ane +Ġl iver +ĠIn side +S orry +Ġagre es +Ġfund ament +ĠF isher +Ġa uction +Ġthread s +gl as +ĠBas ic +ĠN at +Ġlack ing +Ġceleb ration +j u +Ġs illy +E uro +Ġt att +ight y +cont rolled +T est +ĠSing h +Ġr age +Ġrh yth +o ffic +ĠPh antom +Ġhead lines +Ġrespond ing +ĠMor ning +Ġvit amin +Ġboot s +ĠS ite +al in +p i +Ġvir al +ĠU C +D ER +ĠSe x +Ġst ocks +c urrent +Ġch urches +ĠR are +ĠMur phy +Ġden ial +ĠG aming +Ġtou g +Ġn ick +Ġm akers +ĠRon ald +Ġgener ous +ĠD oc +ĠMor ris +Ġtransform ed +ĠN ormal +Ġ10 4 +ĠKick starter +ĠUp on +On line +ĠI RS +Ġw rap +Ġl oving +Ġarri ves +ĠD ue +Ġhe ter +ĠM ade +Ġrent al +Ġbelong s +Ġatt orneys +Ġcro ps +Ġmat ched +ul um +ol ine +10 9 +Ġdis par +Ġbuy ers +ĠCam bridge +Ġeth ics +rou ps +Ġjust ified +Ġmarg inal +Ġrespect ed +win ning +Ġnodd ed +ĠSer ge +ĠForm er +C raft +######## ######## +ĠWar ner +Ġd ash +et e +Ġent ert +ĠE scape +out heast +Ġkn ees +ĠB omb +Ġr ug +P ass +Ġatt itudes +go vernment +ĠPri or +Ġqual ities +Ġnot ification +ĠPh one +l ie +Ġanticip ated +ĠCom bat +ĠBar ry +Ġ198 2 +Us ers +on er +Ġcomput ing +ĠConnect icut +Ġless er +Ġpe ers +ĠC u +Ġtechn ically +Ġsub mission +ĠUn iversal +Ġman ually +our ge +Ġrespond ents +ĠB TC +ĠH ost +Ġf are +ĠB ird +Ġrece ipt +al so +Ġj ack +Ġagric ulture +Ġsk ull +Ġ! = +Ġpass ive +ĠC I +Ġsoc ieties +Ġremind ed +Ġinter ference +B uy +Ġâ ľ +g on +Ġscrut iny +ĠW itch +Ġconduct ing +Ġ ãĥ +Ġexch anges +ĠMit chell +Ġinhab it +Ġtw ist +B D +Ġwhere ver +group on +Ġj okes +ĠBen jamin +ĠR andom +fr ame +ĠL ions +Ġhighlight ed +ĠArk ansas +E nt +Ġp ile +Ġpre lim +g s +mind ed +Ġfel ony +ĠG A +ĠL uck +Ġpract ically +ĠB os +Ġact ress +D am +ĠB ou +Ġvis a +Ġembed ded +Ġhy brid +Ġear liest +Ġsoon er +s ocial +ĠH A +Ġste ep +Ġdis advant +Ġexplo it +ĠE gg +ĠUlt ra +Ġnecess ity +L ocal +ie ge +Ġd ated +Ġmass es +Ġsubsc ription +pl ess +Ġan onym +Ġpresum ably +Bl ue +The ir +asket ball +ĠPhil ip +Ġcom ed +load ed +r ane +Ġref lection +Ch ina +Ġext ends +Ġform ing +Ġund ers +200 1 +Ġgr at +Ġconcent rations +Ġins ulin +Ġsec ular +Ġwh ilst +Ġwin ners +Ad vertisements +Ġdeliber ately +ĠWork ing +Ġs ink +et ics +d ale +Ġmand ate +Ġg ram +Ġvac ation +Ġwarn ings +ri pp +ĠTH AT +Ġcomment ary +Ġint u +Ġa est +Ġreason ing +Ġbreak down +ĠZ ombie +Ġ-- > +ĠPolit ical +c ott +Ġthr ust +Ġtechn ological +Ġdec iding +Ġtraff icking +L ong +W elcome +pr ising +ĠCommun ications +Ġend ors +Ġsw ift +Ġmetab ol +co ins +res a +ĠHT TP +Ġen roll +ĠH appy +us r +int age +Ġ[ " +u ably +ĠM aterial +Ġrepe al +Se pt +k h +ĠMod i +Ġunder neath +ĠI L +sh ore +Ġdiagn osed +ace utical +Ġsh ower +au x +ĠSw itch +ĠStre ngth +Ġj ihad +n ational +Ġtra uma +uss y +on i +Ġcons olid +Ġcal ories +ĠF lynn +ag ged +16 8 +ĠP ink +Ġfulf ill +Ġch ains +Ġnot ably +ĠA V +L ife +ĠCh uck +m us +ĠUr ban +ĠH end +Ġdep osit +ĠS ad +Ġaff air +OR K +ie val +ĠF DA +Ġt rop +ĠOver all +Ġvirt ue +Ġsatisf action +au nd +Ġl un +ĠSw itzerland +ĠOper ation +pro cess +Ġsh ook +Ġcount ies +le ased +ĠCharl otte +1 12 +Ġtrans cript +Ġre dd +p ush +ĠHe y +ĠAn alysis +[ " +Ġaltern atives +ard less +Ġele ph +Ġpre jud +ĠLe af +H aving +ĠH ub +Ġexpress ions +ĠVol ume +Ġshock ing +ĠRed s +Ġread ily +Ġplan ets +ad ata +Ġcollaps ed +ĠMad rid +Ġir rit +i pper +ĠEn c +ĠW ire +Ġbu zz +ĠG P +ash a +Ġaccident ally +ur u +Ġfrust rated +ĠS A +Ġhung ry +ĠH uff +Ġlab els +ant o +ĠE P +Ġbar riers +) | +ĠBer keley +ĠJ ets +Ġp airs +ĠL an +J ames +ĠB ear +Ġhum or +ĠLiber ty +Ġmagn itude +Ġag ing +ĠM ason +Ġfriends hip +umb ling +Ġemer ge +Ġnewsp apers +Ġam bitious +ĠRich ards +atern al +Ġ198 1 +Ġcook ies +Ġsc ulpt +Ġpur suit +L ocation +Ġscript s +p c +Ġarrang ements +Ġd iameter +Ġl oses +am ation +Ġl iqu +ĠJ ake +aret te +Ġunderstand s +ĠZ en +v m +Ġappro ve +Ġw ip +Ġult ra +Ġint end +ĠD I +asc ular +Ġst ays +ĠK or +ĠK l +Ġinvest ing +L a +Ġbelie ving +b ad +m outh +Ġtaxp ayer +ãĥ ĥ +ĠQue bec +Ġl ap +ĠSw iss +d rop +Ġdr ain +ir i +et c +ft en +ĠN ex +Ġst raw +Ġscream ing +Ġcount ed +Ġdam aging +Ġamb assador +cent ury +Ġpro x +Ġarrest s +u v +il ateral +ĠCh arg +Ġpresc ribed +Ġindepend ently +Ġf ierce +ĠB aby +Ġb rave +Ġsu its += > +Ġbas eline +ĠR ate +Ġis lands +Ġ( ( +g reen +ix els +Ġname ly +ĠVill age +th an +am y +V ersion +g mail +ential s +ĠS ud +ĠMel bourne +Ġarri ving +Ġquant um +e ff +rop olitan +T ri +Ġfun eral +ĠI R +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +ĠC ob +it ably +Ġt urb +Ġcomb o +Re view +Ġdeploy ment +u ity +ĠB ott +Ġinv isible +Ġrender ing +Ġunl ocked +Ġa qu +ĠVlad imir +Ġp ad +ĠBr ain +ĠLeg acy +dr agon +ĠKurd ish +Ġsound ed +Ġdet ained +ĠD M +g ary +Ġd aughters +Ġdistur bing +uk a +ĠPar ad +Ġt ast +Ġunf ortunate +Ġu l +em in +Ġattend ance +tr l +Ġpar ks +ĠMem orial +ĠAl ice +oth y +gu ard +ĠD ise +ĠSh an +ĠFor um +R ich +Ġshif ted +ue z +Ġl ighter +ĠMag n +Ġc od +S ch +ham mad +P ub +3 50 +ĠP okemon +Ġprot otype +Ġun re +B ase +ĠStud ents +ĠRep ly +ĠCommun ist +Ġg au +ĠTy ler +I Z +Ġparticip ated +Ġsup rem +ĠDet ails +Ġvessel s +ro d +Ġt ribe +ke ep +Ġassum ptions +Ġp ound +Ġcr ude +ĠAv ailable +Ġswim ming +Ġin clusion +Ġadv ances +c ulation +Ġconserv ation +Ġover d +ĠBuff alo +Art icle +ed ge +Ġaw a +ĠMad ison +Ġsid ew +Ġcat ast +ĠK rist +uc le +ĠHigh way +ĠTer ror +Ġactiv ation +Ġuncons cious +ĠSat an +ĠSus an +ill ery +Ġarr anged +i op +Ġrum ors +ur ring +th ink +ĠKe ith +ĠK ind +Ġavoid ing +by n +n ut +ĠSpe aker +r us +n ames +Ġgu ilt +ĠOlymp ics +Ġsa il +ĠM es +lev ant +ĠColumb us +a ft +C ity +S outh +ĠHar vey +ĠP un +S everal +Ġment ally +Ġimp ress +m ount +ĠUb untu +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +ĠSuper man +ĠMP s +Ġintent ions +ĠR acing +Ġlike lihood +Ġ2 40 +T otal +Ġto ys +ĠW atson +Ġur ge +L ear +ĠP aper +Ġoccur ring +ĠB eng +ĠC ert +Ġst ones +T im +ĠTw in +z b +ĠD ynam +Ġpolit ician +k ens +ĠEnter prise +UT ERS +Ġab ol +Ġref resh +Ġarbit rary +pe ction +Ġtrou bles +Ġ} ); +t v +Ġpil ots +Ġdist ribute +Ġaud it +Ġp ause +orig inal +Ġr ivals + £ +F ig +T L +ab il +ry ing +L in +ion ed +l on +Ġf ancy +Ġcr ashed +Ġt ract +Ġshe d +Ġcons ume +B ased +down load +in it +Ġvolt age +Int rodu +Ġcondem ned +ĠFin ance +res pect +Ġex cluded +Ġestablish ing +her ic +Ġher itage +Ġspect acular +Ġun st +ĠSnow den +ĠL ane +S an +Ġprotect ions +st ruction +inc inn +Ġmac ro +C ustom +ios ity +Ġes p +Ġfunction ing +Ġm ush +Ġp uzzle +Ġeth ical +M al +Ġgo verning +ĠF erguson +Ġrest ored +Ġst ressed +ĠCoun ter +ĠK as +cl ip +AN S +Ġse iz +U K +by ss +old own +ap i +Ġperman ently +oun ters +W est +Th rough +L ight +at oes +Ġne at +Ġc ord +ure r +Ġsevere ly +ĠA ven +Ġinter rog +Ġtri ple +G iven +N umber +Ġar ise +Ġs her +pl ant +Ġfl ower +ĠC ou +Ġat e +Ġnew er +b ul +Ġmean while +ĠL air +Ġadjust ment +ĠCop yright +Ġd ivers +i ological +Ġgam ers +o at +Ġhistor ically +Ġanal og +Ġlong time +Ġpres cription +ĠM ist +ĠHy per +ĠM aine +ĠDe ity +Ġmulti pl +ĠRe incarn +ĠH yd +ĠP ic +S il +r ants +ĠC ris +. ; +( { +epend ence +Ġrec y +ate ur +Ġqu ad +Ġgl ob +Ġcon ced +te am +Ġcapital ist +ĠL ot +Ġroy al +ĠCy ber +Ġblack s +met ic +ri v +ĠD anny +Ġsp o +ĠR O +Ġanim ated +rypt ed +ĠDep uty +Ġrend ered +F E +Ġstre ak +Ġcloud s +ĠDou g +~~~~ ~~~~ +Ġdisc our +ĠVe h +Ġpsych ology +ĠJ ourney +Ġcry stal +ĠFro st +Ġsuspic ion +Ġrel ate +or us +ĠC rypt +ĠN VIDIA +com ed +ut ing +incinn ati +Ġvulner ability +ost ic +Ġisol ation +Ġcool ing +ĠCoal ition +Ġ1 19 +F our +ĠDe al +Ġâ ī +se mble +ram ent +ĠBar celona +Ġ10 2 +Ġcoc aine +ocaly pse +F eb +ogen ic +Ġmut ation +Ġcrypt oc +ĠK el +ĠG it +a is +Ġs isters +AN K +Ġactiv ate +T er +Ġd read +yl on +Ġprop ri +A ust +ĠDef ault +Ġout door +Ġshe er +ce ive +Ġg ently +Ð ¾ +Pro gram +Ġâ ĨĴ +Ġve gan +ĠCr us +Ġrespons ibilities +ĠH R +OL D +Ġprev ents +Ġst iff +ĠW ere +Ġathlet ic +ĠSc ore +Ġ) : +Ġcolumn s +ĠL oc +av ailable +ĠF ram +ĠS essions +Ġcompan ion +Ġpack s +14 0 +ĠKn ights +Ġf art +Ġstream s +Ġsh ore +Ġapp eals +ĠPer formance +h aul +ĠSt ra +ĠN ag +10 3 +ĠTrans portation +B B +E v +z an +P ublic +Ġtw in +uls ion +M ult +Ġelect ro +Ġstat ue +ation ally +ĠN ort +Ġins pection +/ * +ig ue +Ġcomp assion +ĠT ales +ĠSte in +ĠSc reen +ĠB ug +ĠL ion +g irl +Ġwithdraw al +Ġobject ives +Ġblood y +Ġprelim inary +Ġj acket +Ġdim ensions +ĠC ool +ĠOcc up +Ġw reck +Ġdoub led +ank ing +Ġ19 75 +Ġglass es +ĠW ang +pro v +P ath +connect ed +ĠMult i +ĠNor way +agon ist +Ġfe ared +Ġtouch ing +Ġarg uably +¯¯¯¯ ¯¯¯¯ +ĠNC AA +che m +Ġsp at +ĠW WE +ĠC el +ig ger +Ġattack er +ĠJo in +ob ject +ett a +Ġelim inated +d et +Ġdest ruct +ĠLuc as +ct uary +18 0 +ĠBr ady +ĠBl ues +B ay +au kee +Ġtim eline +Ġdeleg ates +w ritten +uff icient +Ġsh apes +Cop yright +ou ble +serv ice +Ġp ione +Ġcolleg es +Ġrow s +Ġsp ite +Ġassess ed +3 60 +Ġle ase +Ġconfident ial +ck er +ĠMan ning +ĠV oice +Ġse aled +Ġcalcul ate +N O +ĠAss istant +Ġteen ager +ul ent +ather ine +Ġm ock +Ġd iamond +Ġf est +Ġsw itched +Ġres ume +ĠPu erto +Ġl anes +ir ation +ĠSimilar ly +Ġro d +ĠS el +ĠPal ace +ĠLim ited +e ous +Ġvar iant +Ġw ard +Ġ) ) +Sh ow +OO K +A lex +ĠN ep +br is +ĠWik ipedia +Ġexcept ional +Ġman ages +ĠD raw +Ag ain +Ġco pper +ut t +Ġex ports +Ġport folio +Ġelev ated +R ated +ĠOther wise +ĠT act +ĠShe l +ĠT X +" âĢĶ +Ġres ur +ĠW a +ven ant +Ġmon etary +pe ople +E mail +Ġfif ty +ĠS weet +ĠMalays ia +Ġconf using +ĠR io +ud a +uten ant +" ); +Ġpra ised +Ġvol umes +t urn +Ġm ature +Ġnon profit +Ġpassion ate +ĠPriv ate +Ġ10 3 +Ġdesc end +ç ¥ŀ +uff y +head ed +Whe ther +ri en +ze ch +be it +Ġch rom +ĠMc M +Ġd ancing +Ġe leg +ĠNot iced +11 5 +Ġadvoc acy +ENT S +amb ling +ĠMin or +ĠF inn +Ġprior ities +Ġthere of +ĠSt age +ĠRog ers +Ġsubst itute +ĠJ ar +ĠJeff erson +Ġlight ly +10 2 +ĠL isa +u its +ys ical +Ġshif ts +Ġd rones +Ġwork place +Ġres id +ens ed +ah n +Ġpref erences +ser ver +Ġdeb ates +d oc +ĠGod s +Ġhelicop ter +Ġhon our +Ġconsider ably +ed ed +ĠF emale +ĠAn ne +Ġre un +ĠF ace +ĠHall ow +ĠBud get +Ġcondem n +Ġt ender +Pro f +ocr atic +ĠTurn er +ĠAg ric +Ġ19 76 +Ġa pt +d isc +ĠF ighter +ĠA ur +Ġgar bage +in put +ĠK arl +ĠOl iver +ĠL anguage +k n +N on +ĠCl ar +Ġtrad itions +Ġad vertisement +ĠS or +Ġarch ive +Ġvill ages +7 50 +Ġimplement ing +w aukee +Ġdiet ary +Ġswitch ing +Rep ublic +Ġvel ocity +Ġc it +ĠA wards +Ġfin ancing +Ġlast ed +) ] +Ġrem inder +P erson +Ġprec ision +Ġdesign ers +ĠF ried +ĠB order +Ġtr agic +Ġw ield +Ġiniti atives +ĠT ank +w er +Ġjo ins +R o +in ery +Ġar row +Ġgener ating +found er +Ġsear ches +Ġrandom ly +A ccess +Ġb atch +Ġp osed +l at +Ġpursu ing +as a +Ġtest ified +form ing +ĠSh ar +w iki +ĠE ither +S ometimes +Ġsen ators +ĠJohn ny +ĠTal iban +ĠG PS +":" / +ãģ® å +Ġanaly zed +ĠRub io +ĠMove ment +op ard +ii i +St and +f ight +Ġign oring +i ang +ĠG N +so ever +ĠST AT +Ġref using +Ġswe at +Ġb ay +P ORT +ir med +ak y +Ġdis pro +Ġlabel ed +Ġ10 8 +H ello +Ġple asant +ab a +Ġtri umph +Ġab oard +Ġinc om +ĠC row +le tt +Ġfol k +Ġch ase +` ` +ĠBr us +Ġte ens +c ue +Ġter rain +h yd +il ight +OR Y +Su pport +ew s +ll i +rain ts +ĠC and +Ġab used +ach ment +l arg +B as +ĠC ancer +Ġ19 78 +Ġsupp orter +ac cess +ĠTer min +ĠT ampa +ĠAN Y +Ġnew est +ĠCrim inal +ed u +Ġ19 30 +Ġadm its +Ġend e +Ġfail ures +ur ate +ful ness +cy cl +ĠSub ject +Ġinf inite +th ree +W A +p it +ĠInst all +R ad +ili ation +G M +Ġcontin ent +Ġaccommod ate +ĠCl ay +Ġp up +ĠF unction +Ġham mer +ĠAlbert a +Ġrev ised +Ġminor ities +Ġmeasure ment +Con nell +Ġdis able +ĠM ix +In cre +Ġfor k +ĠR osen +Ġimpl ies +umb lr +AN G +Ġprote ins +Ġagg ression +Ġfacilit ate +S N +Ġilleg ally +u er +Ġacad em +Ġp uzz +ĠSh ift +p ay +oll o +Ġaud iences +B uild +Ġno ble +Ġsynt ax +â ĺħ +Ġbe am +ĠB ed +ĠA ld +Ġorig ins +v ideo +Ġ19 77 +ĠAss ault +Ġgar age +Te am +Ġver dict +Ġd war +ĠVirt ual +e vent +Ke ep +Ġsent iment +Ġwild life +sh irt +Ġb urg +Ġrecommend ation +rep resent +Ġgall ery +own ers +Ġsch olar +Ġconven ience +ĠSw ift +Ġconv inc +C ap +Ġwar fare +ĠVis ual +Ġconst itute +Ġab ort +ĠWe ather +ĠLook ing +ĠH em +Ġmart ial +Ġinc oming +et ition +Ġtoler ance +ĠCre ated +Ġfl ows +ĠE lder +Ġsoul s +Ġf oul +ĠP ain +ĠC AN +Ġ2 20 +b c +he nd +Ġgen ius +R eal +ĠW r +omet er +p ad +Ġlim iting +ĠS i +ĠL ore +ĠAd ventures +Ġvar ied +D isc +f in +ĠPerson al +Ch ris +Ġinv ented +Ġd ive +ĠR ise +Ġo z +ĠCom ics +Ġexp ose +ĠRe b +let ters +s ite +im ated +Ġh acking +Ġeduc ated +ĠNob ody +Ġdep ri +Ġincent ive +ãĤ · +Ġovers ight +Ġtrib es +ĠBelg ium +Ġlicens ing +our t +Produ ct +ah l +ĠG em +Ġspecial ist +Ġc ra +ann ers +ĠCor byn +Ġ19 73 +RE AD +Ġsum mar +Ġover look +ĠApp lication +Ġin appropriate +Ġdownload ed +Q ue +ĠB ears +Ġth umb +ĠChar acter +ĠReincarn ated +ĠS id +Ġdemonstr ates +s ky +ĠBloom berg +ĠAr ray +ĠRes ults +ĠFour th +ĠED T +ĠO scar +c end +Ġ10 6 +ĠN ULL +ĠH ERE +m atch +ĠBr un +Ġgluc ose +ie g +eg u +Ġcert ified +Ġrel ie +Ġhuman itarian +Ġpr ayers +K ing +Ġn an +h ou +10 8 +ul u +Ġrenew able +Ġdistingu ish +Ġd ense +ĠV ent +ĠPack age +ĠB oss +Ġedit ors +Ġm igr +T ra +ĠPet ers +ĠAr ctic +200 4 +ĠC ape +Ġloc ally +Ġlast ing +Ġhand y +. ). +P an +ĠR ES +Ind ex +Ġt ensions +Ġformer ly +Ġide ological +Ġsens ors +Ġdeal ers +Ġdef ines +S k +Ġproceed s +Ġpro xy +az ines +ĠB ash +ĠP ad +ĠC raft +eal ous +Ġshe ets +omet ry +J une +cl ock +T T +ĠThe atre +ĠB uzz +Ġch apters +Ġmill enn +Ġd ough +ĠCongress ional +Ġimag ined +av ior +Ġclin ic +Ġ19 45 +Ġhold er +ro ot +oles ter +Ġrest art +B N +ĠHam as +ĠJ ob +Ġor b +Ġr am +Ġdiscl ose +Ġtransl ate +Ġimm igrant +Ġannoy ing +Ġtreat y +an ium +ĠTe a +ĠLeg ion +Ġcrowd s +ĠB ec +ĠA er +oh yd +B ro +Look ing +Ġl bs +Ġagg ress +Ġse am +Ġinter cept +ĠM I +mer cial +act iv +ĠC it +Ġdim ension +Ġconsist ency +Ġr ushing +ĠDou glas +Ġtr im +Inst all +ick er +Ġsh y +10 6 +Ġment ions +pe lled +ĠT ak +c ost +Ġclass room +Ġfort une +dri ven +Ġun le +ĠWhe el +Ġinvest or +ĠM asters +k it +Ġassoci ations +ĠEv olution +op ing +us cript +Ġprov incial +ĠWal ter +av i +S O +Ġun limited +Eng lish +ĠC ards +ĠEb ola +ne red +Ġreven ge +Ġout right +um per +Ġf itting +ĠSol id +Ġform ally +Ġproblem atic +Ġhaz ard +Ġenc ryption +Ġstraight forward +ĠA K +Ġp se +ĠOr b +ĠCh amber +ĠM ak +Cont ents +Ġloyal ty +Ġl yrics +ĠSy m +Ġwel comed +Ġcook ed +Ġmon op +Ġn urse +Ġmis leading +Ġe ternal +Ġshif ting +Ġ+ = +V is +Ġinst itutional +ill ary +Ġp ant +VER T +ĠA CC +ĠEn h +Ġinc on +ĠRE UTERS +Ġdon ated +â̦â̦ â̦â̦ +In tern +Ġexhib it +Ġt ire +ĠR ic +ĠCh ampion +ĠMu hammad +N ING +ĠSoc cer +Ġmob ility +Ġvary ing +ĠM ovie +Ġl ord +o ak +F ield +Ġve ctor +us ions +Ġsc rap +Ġen abling +m ake +T or +. * +| | +ĠWe bsite +ĠN PC +Ġsocial ist +ĠBill y +ĠAdd itional +Ġc argo +Ġfar ms +ĠSo on +ĠPri ze +Ġmid night +Ġ9 00 +se en +ĠSp ot +Ġshe ep +Ġspons ored +ĠH i +ĠJ ump +Ġ19 67 +Micro soft +ĠAg ent +Ġch arts +d ir +Ġadj acent +Ġtr icks +Ġman ga +Ġex agger +/ > +foot ball +ĠF CC +G C +ĠT ier +and ra +OU ND +% ), +Ġfru its +V C +ĠA A +R ober +Ġmid st +â Ĺ +ank a +Ġlegisl ature +ĠNe il +Ġtour ists +" " +ĠWar ning +ĠNever theless +ĠOffic ial +ĠWh atever +Ġm old +Ġdraft ed +Ġsubst ances +Ġbre ed +Ġt ags +ĠT ask +Ġver b +Ġmanufact ured +com ments +ĠPol ish +Pro v +Ġdetermin es +Ob ama +k ers +Ġutter ly +Ġse ct +sc he +ĠG ates +ĠCh ap +Ġal uminum +Ġz ombie +ĠT ouch +ĠU P +Ġsatisf y +Ġpred omin +asc ript +Ġelabor ate +Ġ19 68 +Ġmeas uring +ĠV ari +any ahu +Ġs ir +ul ates +id ges +ick ets +ĠSp encer +T M +oub ted +Ġpre y +Ġinstall ing +ĠC ab +re ed +re ated +Su pp +Ġwr ist +ĠK erry +10 7 +ĠK le +ĠR achel +Ġc otton +ĠA RE +ĠE le +Cont rol +Ġload s +ĠD od +an as +b one +Ġclass ical +ĠReg ional +ĠInt eg +V M +Ġdes ires +Ġaut ism +support ed +ĠM essage +Ġcomp act +writ er +Ġ10 9 +ĠHur ricane +c ision +Ġcy cles +Ġdr ill +Ġcolle ague +Ġm aker +G erman +Ġmist aken +S un +ĠG ay +Ġwhat soever +Ġsell s +ĠA irl +l iv +ĠO ption +Ġsol ved +Ġse ctors +Ġhorizont al +Ġequ ation +ĠSk ill +ĠB io +g ement +ĠSn ap +ĠLeg al +Ġtradem ark +Ġmake up +Ġassemb led +Ġsa ves +ĠHallow een +ĠVer mont +ĠFR OM +Ġfar ming +ĠP odcast +accept able +ĠHig her +Ġas leep +ull ivan +Ġrefere n +ĠLe v +Ġbul lets +ok o +H C +Ġst airs +Ġmain tains +ĠL ower +ĠV i +Ġmar ine +Ġac res +Ġcoordin ator +ĠJ oh +Ġcounterpart s +ĠBrother s +Ġind ict +b ra +Ġch unk +Ġc ents +H ome +ĠMon th +Ġaccording ly +if les +ĠGerm ans +ĠSy n +H ub +Ġey eb +âĶĢâĶĢ âĶĢâĶĢ +Ġr anges +ĠHoll and +ĠRob ot +f c +M ike +Ġpl asma +Ġsw ap +Ġath lete +ĠR ams +,' " +Ġinfect ions +Ġcor rid +Ġv ib +Ġpat ches +Ġtradition ally +Ġrevel ation +Ġswe ep +Ġgl ance +Ġin ex +200 3 +ĠR aw +work ing +os ures +ĠD at +ĠLyn ch +Ġle verage +ĠRe id +Ġcorrel ation +ian ces +av ascript +Ġrep ository +ret ty +Ġ19 72 +24 0 +Ġo un +p ol +ĠRe ed +Ġtact ical +is ite +App le +ĠQu inn +Ġrap ed +ill o +Euro pe +Ġalgorith ms +ĠRod rig +i u +Ġill um +Ġf ame +Ġintrodu cing +Ġdel ays +ĠRaid ers +Ġwh istle +Ġnovel s +ĠRe ally +Ġder iv +Ġpublic ations +ĠNe ither +ĠCom merce +Ġa ston +l anguage +Not es +ĠR oth +ĠF ear +Ġm ate +Ġpar ade +ĠQ B +Ġman eu +ĠC incinnati +m itting +Ġwa ist +ĠR ew +Ġdisc ont +Ð ° +Ġst aring +Ġal ias +Ġsec urities +Ġtoile t +ĠJ edi +Ġun law +v ised +//// //// +] ( +ĠWe iss +Ġpre st +ĠComp an +Ġmem o +ĠGr ace +J uly +ĠEl ite +cent er +ĠSt ay +Ġgal axy +Ġto oth +ĠS ettings +Ġsubject ed +ãĤ ¦ +Ġline back +Ġretail ers +ĠW ant +Ġd angers +A ir +Ġvolunt ary +ew ay +Ġinterpret ed +ot ine +à § +Ġp el +Serv ice +ĠEvent ually +Ġcare ers +Ġthreat en +Ġmem or +ĠBrad ley +anc ies +s n +ĠUn known +N ational +Ġsh adows +ail and +ĠD ash +Every one +izz ard +M arch += ( +Ġpull s +Ġstr anger +Ġback wards +ĠBern ard +imens ional +Ġch ron +Ġtheoret ical +k top +Ġw are +ĠInvest ig +ĠIn iti +ĠOper ations +o ven +oc ide +* / +Ġfl ames +ĠC ash +sh it +Ġc ab +ĠAn aly +ĠSe ah +Ġdefin ing +Ġorder ing +Ġimm un +Ġpers istent +AC H +Russ ian +m ans +Ġh ind +Ġphot ography + © +Ġh ug +Ġ10 7 +ĠH ence +i ots +ude au +Ġsubsid ies +Ġroutine ly +ĠDev ice +it ic +Ġdisg ust +land er +Ġ19 40 +Ġassign ment +ĠB esides +w ick +ĠD ust +us c +struct ed +11 1 +de velop +Ġf ond +Ġinter section +Ġdign ity +Ġcommission er +With out +re ach +Ġcart oon +Ġsc ales +ãĥ Ń +F IG +Ġsurve ys +ĠIndones ia +Ġart work +Ġun ch +Ġcy cling +un ct +au er +or ate +ĠOb viously +Ġcharacter ized +fe ld +Ġaff irm +Ġinn ings +Ġ é +Ġal iens +Ġcl oth +et ooth +ĠC ertain + § +Ġdig est +k now +ĠX L +Ġpredict ions +Ġd in +W AR +Ġafter math +Ex ample +ĠSu ccess +ĠTh r +IG N +Ġmin er +B us +Ġcl arity +heim er +ĠO UT +ĠS end +ĠCirc le +ĠD iet +Ġpron ounced +Ġcreat ors +Ġearthqu ake +atter y +ge ons +Ġo d +Ġlay ing +or p +U lt +pro ject +Ġunder min +Ġsequ el +S am +ĠDark ness +Ġre ception +b ull +Y S +ĠV ir +Ġsequ ences +ĠCo in +Ġout fit +ĠW ait +1 19 +Ġdel ivers +.... .. +Ġbl own +ĠE sc +ĠM ath +per m +ĠU l +Ġgl im +Ġfac ial +Ġgreen house +Ġto kens +/ - +ĠAnn ual +ĠON E +Ġteen age +ĠPhys ical +ĠL ang +ĠC elt +Ġsu ed +ivid ually +Ġpat ience +ch air +reg ular +Ġa ug +in v +ex cept +ĠL il +Ġn est +f d +s um +ĠCh ase +Russ ia +ĠJenn ifer +Ġoff season +Over all +F ore +Ġr iot +A ud +form er +Ġdefend ers +ĠC T +iot ic +rib ly +Ġautom ated +Ġpen is +Ġins ist +Ġdi agram +ĠS QL +ĠG arc +Ġw itch +cl ient +ier ra +am bers +Ġrec ount +f ar +V ery +oster one +Ġappreci ated +ĠPer fect +S ection +Ġd oses +oca ust +Ġcost ly +Ġg rams +ĠSh i +Ġwrest ling +Ġ19 71 +Ġtro phy +Ġn erve +ĠK az +ĠExper ience +Ġpled ged +Ġplay back +Ġcreat ivity +by e +Ġattack ers +Ġhold ers +ĠCo ach +ĠPh D +Ġtransf ers +Ġcol ored +ĠH indu +Ġd rown +Ġlist ened +ĠW A +ias m +P O +Ġappeal ing +Ġdiscl osed +ĠCh icken +ag ging +Ġple aded +Ġnav igation +ĠReturn s +Ġ[ [ +R OR +E A +Ġphotograp her +ĠR ider +ipp ers +Ġsl ice +Ġe rect +Ġhe d +iss ance +ĠVik ings +ur ious +Ġapp et +oubted ly +Ch ild +Ġauthent ic +o os +ĠM aking +Ġannoun cing +Ġb od +Ġmet er +ĠN ine +ĠR ogue +Ġwork force +Ġrenew ed +Ġorganis ations +ac s +P LE +Sh ort +Ġcomp ounds +ĠVis it +Ġen velop +ear th +Ġsupport ive +gg le +ĠBrus sels +ĠGu ild +Cre ate +RE L +Ġaver aged +Ġ19 69 +ri ages +Ġlength y +Ġforg ot +O kay +ĠE rd +Ġdeal er +Ġrec ession +D D +Ġdesper ately +Ġhun ger +Ġst icks +Ġm ph +ĠF aith +Ġintention ally +Ġdem ol +ue ller +ĠS ale +Ġde bris +s pring +Ġle ap +>> >> +Ġcontain ers +se lling +rane an +atter ing +Ġcomment ed +ĠC M +on ut +Ġwood s +es pecially +Ġorgan ize +iv ic +ĠWood s +ang a +s qu +Ġm aj +am on +Ġax is +Ġ19 74 +ĠDen mark +Ġwar rior +ĠP and +Ġout lined +ĠB O +ins ula +z illa +eb ook +Ġd are +Ġsear ched +Ġnav igate +S n +writ ing +Ġun ited +J apan +ĠHe brew +Ġfl ame +Ġrel ies +Ġcatch ing +ĠSh o +Ġimprison ment +Ġp ockets +Ġclos ure +ĠF am +t im +ade qu +Act ivity +Ġrecru iting +ĠW ATCH +ĠArgent ina +d est +Ġapolog ize +or o +Ġlack s +Ġtun ed +ĠGriff in +Ġinf amous +Ġcelebr ity +ss on +Ġ ---------------------------------------------------------------- +ĠIs is +ĠDis play +Ġcred ibility +Ġeconom ies +Ġhead line +ĠCow boys +Ġind ef +Ġl ately +Ġincent ives +but ton +ĠM ob +A ut +Ġres igned +ĠO m +c amp +Ġprof iles +Ġsche mes +olph ins +ay ed +Cl inton +en h +ĠY ahoo +Ġab st +Ġan k +su its +Ġw ished +ĠMar co +udd en +Ġsp here +ĠB ishop +Ġincorpor ated +ĠPl ant +11 4 +Ġh ated +p ic +Ġdon ate +Ġl ined +Ġbe ans +Ġsteal ing +Ġcost ume +Ġsher iff +Ġfor ty +Ġint act +Ġadapt ed +Ġtrave lling +b art +Ġnice ly +Ġdri ed +Ġsc al +os ity +NOT E +ĠB h +ĠBron cos +ĠI gn +Ġint imate +Ġchem istry +Ġopt imal +D eb +ĠGener ation +Ġ] , +ich i +ĠW ii +ĠYOU R +vent ions +W rite +Ġpop ul +un ning +ĠW or +V ol +Ġqu een +head s +K K +Ġanaly ze +op ic +ear chers +Ġd ot +leg raph +ast ically +Ġupgr ades +Ġca res +Ġext ending +Ġfree ze +Ġin ability +Ġorg ans +Ġpret end +Ġout let +11 3 +ol an +ĠM all +ul ing +t alk +Ġexpress ing +ĠAl ways +ĠBe gin +f iles +Ġlic enses +% % +ĠM itt +Ġfil ters +ĠMil waukee +G N +Ġunf old +M o +Ġnut rition +pp o +B o +Ġfound ing +Ġunder mine +Ġeas iest +ĠC zech +ĠM ack +Ġsexual ity +ĠN ixon +W in +ĠAr n +ĠK in +ãĤ £ +ic er +Ġfort un +Ġsurf aces +agh d +Ġcar riers +ĠP ART +ĠT ib +Ġinter val +Ġfrust rating +ĠSh ip +ĠAr med +ff e +Ġbo ats +ĠAb raham +in is +Ġsu ited +th read +i ov +ab ul +ĠVenezuel a +Ġto m +su per +Ġcast le +alth ough +iox ide +ec hes +Ġevolution ary +Ġnegoti ate +Ġconfront ed +Rem ember +Ġ17 0 +S uch +Ġ9 11 +m ult +ĠA byss +ur ry +ke es +spe c +ĠBarb ara +Ġbelong ing +Ġvill ain +ist ani +Ġaccount able +Ġport ions +ĠDe cl +U r +ĠK ate +g re +Ġmag azines +UC K +Ġregul ate +om on +ĠAl most +Ġover view +Ġsc ram +Ġl oot +ĠF itz +Ġcharacter istic +ĠSn ake +s ay +ĠR ico +Ġtra it +ĠJo ined +au cus +Ġadapt ation +ĠAirl ines +Ġarch ae +ĠI de +Ġb ikes +Ġliter ary +Ġinflu ences +ĠUs ed +C reat +Ġple a +ĠDef ence +ĠAss ass +Ġp ond +UL T +) " +Ġeval uated +Ġob taining +Ġdem ographic +Ġvig il +ale y +Ġsp ouse +ĠSeah awks +resp ons +ĠB elt +um atic +Ġr ises +run ner +ĠMichel le +Ġpot ent +r ace +ĠP AC +F ind +olester ol +IS S +ĠIntrodu ced +ress es +ign ment +O s +ĠT u +ĠDe x +ic ides +Ġspark ed +ĠLaur a +ĠBry ant +Ġsm iling +ĠNex us +Ġdefend ants +ĠCat al +Ġdis hes +sh aped +Ġpro long +m t +( $ +ãĢ Ĥ +Ġcalcul ations +ĠS ame +Ġp iv +H H +Ġcance lled +Ġgr in +Ġterrit ories +ist ically +C ome +ĠP arent +Pro ject +Ġneg lig +ĠPriv acy +Ġam mo +LE CT +olute ly +ĠEp ic +Ġmis under +w al +Apr il +m os +path y +ĠC arson +Ġalbum s +ĠE asy +Ġpist ol +< < +Ġ\ ( +t arget +hel p +Ġinter pre +cons cious +ĠH ousing +ĠJ oint +12 7 +Ġbe ers +s cience +ĠFire fox +effect ive +ĠC abin +ĠO kay +ĠApp lic +Ġspace craft +ĠS R +ve t +ĠStr ange +S B +Ġcor ps +iber al +e fficient +Ġpreval ence +Ġeconom ists +11 8 +Th read +ord able +OD E +ĠC ant +=- =- +if iable +ĠA round +Ġpo le +Ġwilling ness +CL A +ĠK id +Ġcomple ment +Ġsc attered +Ġin mates +Ġble eding +e very +Ġque ue +ĠTr ain +Ġh ij +Ġme lee +ple ted +Ġdig it +Ġg em +offic ial +Ġlif ting +Ð µ +Re qu +it utes +Ġpack aging +ĠWork ers +h ran +ĠLeban on +ol esc +Ġpun ished +ĠJ uan +Ġj am +ĠD ocument +Ġm apping +ic ates +Ġinev itably +Ġvan illa +ĠT on +Ġwat ches +Ġle agues +Ġiniti ated +deg ree +port ion +Ġrec alls +Ġru in +Ġm elt +I AN +Ġhe m +Ex p +Ġb aking +ĠCol omb +at ible +Ġrad ius +pl ug +ĠI F +et ically +Ġf ict +H ER +ĠT ap +atin um +Ġin k +Ġco h +ĠW izard +b oth +te x +Ġsp ends +ĠCurrent ly +ĠP it +Ġneur ons +ig nt +Ġr all +Ġbus es +b uilding +Ġadjust ments +Ġc ried +ibl ical +att ed +ĠZ ion +ĠM atter +Ġmed itation +ĠD ennis +Ġour s +ĠT ab +Ġrank ings +ort al +Ġad vers +Ġsur render +ĠG ob +ci um +om as +im eter +Ġmulti player +Ġhero in +Ġoptim istic +Ġindic ator +ĠBr ig +Ġgro cery +Ġapplic ant +ĠRock et +v id +Ex ception +p ent +Ġorgan izing +Ġenc ounters +ĠT OD +Ġjew el +S ave +ĠChrist ie +Ġhe ating +Ġl azy +ĠC P +Ġcous in +Con fig +Ġreg ener +Ġne arest +Ġachie ving +EN S +th row +ĠRich mond +ant le +200 2 +Ġan ten +b ird +13 3 +Ġn arc +r aint +un ny +ĠHispan ic +ourn aments +Ġprop he +ĠTh ailand +ĠT i +Ġinject ion +Ġinher it +rav is +Ġmed i +Ġwho ever +ĠDE BUG +G P +ĠH ud +C ard +p rom +Ġp or +Ġover head +L aw +Ġviol ate +Ġhe ated +Ġdescript ions +Ġachieve ments +ĠBe er +ĠQu ant +W as +Ġe ighth +ĠI v +Ġspecial ized +U PDATE +ĠD elta +P op +J ul +ĠAs k +oph y +Ġnews letters +ĠT ool +Ġg ard +ĠConf eder +ĠGM T +ĠAb bott +Ġimm unity +ĠV M +Is lam +Ġimpl icit +w d +Ġ19 44 +rav ity +omet ric +Ġsurv iving +ur ai +ĠPr ison +Ġr ust +ĠSk etch +Ġbe es +ĠThe ory +Ġmer it +T ex +ch at +Ġm im +Ġpast e +ĠK och +Ġignor ance +ĠSh oot +Ġbas ement +Un ited +ĠAd vis +he ight +Ġf oster +Ġdet ain +in formation +Ġne ural +' ; +Ġprov es +all ery +Ġinv itation +um bers +Ġc attle +Ġbicy cle +z i +Ġconsult ant +Ġap ology +ĠT iger +Ġ12 3 +99 9 +Ġind ividually +r t +ig ion +ĠBrazil ian +Ġdist urb +Ġentreprene urs +Ġfore sts +cer pt +pl ates +p her +clip se +Ġtw itter +Ġac ids +ograph ical +h um +ĠB ald +if ully +Ġcomp iler +ĠD A +Ġdon or +as i +Ġtrib al +l ash +ĠCon fig +Ġapplic ants +Ġsal aries +13 5 +Put in +ĠF ocus +ir s +Ġmisc onduct +ĠH az +Ġeat en +M obile +Mus lim +ĠMar cus +v iol +Ġfavor able +Ġst ub +ad in +ĠH ob +Ġfaith ful +Ġelectron ics +Ġvac uum +w ait +back ed +econom ic +d ist +Ġten ure +Ġsince re +ĠT ogether +ĠW ave +Ġprog ression +Ġden ying +Ġdist ress +br aska +th ird +Ġmix ing +Ġcolon ial +Ġpriv ately +Ġun rest +atern ity +Ġprem ises +ant i +greg ation +Ġlic ence +ĠH ind +ĠSam uel +Ġconvinc ing +ĠA ce +ĠR ust +ĠNet anyahu +Ġhand les +ĠP atch +orient ed +ah o +ĠG onz +Ġhack ers +claim er +Ġcustom s +ĠGr an +f ighters +Ġl uc +Ġman uscript +aren thood +Ġdev il +Ġwar riors +Ġoff enders +Will iam +Ġhol idays +Ġnight mare +Ġle ver +iff erent +St at +Ġexhib ition +put ed +ĠP ure +Ġal pha +Ġenthus iasm +ĠRepresent atives +E AR +ĠT yp +Ġwhe at +ĠAl f +Ġcor rection +Ġev angel +AT T +M iss +Ġs oup +Ġimpl ied +par am +Ġsex y +ĠL ux +Ġrep ublic +p atch +ab lish +Ġic ons +Ġfather s +ĠG ET +ĠCar ib +Ġregul ated +ĠCo hen +ĠBob by +Ġn er +Ġb ent +vent ory +ĠAl ong +ĠE ST +ĠWall ace +Ġmurd ers +r ise +ke ll +ĠCommon wealth +Ġn asty +et a +ĠM IT +Ġadminist ered +Ġgenuine ly +Ed itor +n ick +Ġhyd ro +**************** **************** +ĠB le +Ġfin es +Ġg orge +aus ible +r h +Ġapp le +ment ioned +Ġro pe +ot yp +H R +Ġdisappoint ing +Ġc age +n ik +Ġdoub ts +ĠF REE +print s +ĠM UST +Ġvend ors +ĠIn qu +Ġliber als +Ġcontract or +Ġup side +child ren +Ġtrick y +Ġregul ators +charg ed +l iter +Ġ *** +Ġreb ell +l ang +Ġloc als +Ġphys icians +Ġhe y +ar se +t m +ĠLe x +Ġbehavior al +success ful +F X +Ġbr ick +ov ic +Ġcon form +Ġreview ing +Ġins ights +Ġbi ology +ĠRem ove +ĠExt ra +Ġcomm itting +indu ced +ignt y +ig m +Ġat omic +Comm on +ĠE M +ĠP ere +ĠIt ems +e h +Ġpres erved +ĠH ood +Ġprison er +Ġbankrupt cy +Ġg ren +us hes +Ġexplo itation +Ġsign atures +Ġfin an +] ," +ĠM R +Ġme g +rem lin +Ġmusic ians +Ġselect ing +Ġexam ining +IN K +l ated +H i +Ġart ic +Ġp ets +Ġimp air +ĠM AN +Ġtable ts +in clude +R ange +Ġca ut +Ġlog s +Ġmount ing +Ġun aware +Ġdynam ics +ĠPalest ine +ĠQu arter +ĠPur ple +Ġm a +ĠIm port +Ġcollect ions +ci ation +Ġsuccess or +Ġcl one +Ġaim ing +Ġposs essed +Ġstick ing +Ġsh aking +Ġloc ate +ĠH ockey +T urn +17 0 +Ġfif teen +ĠHar rison +Ġcontinu ously +ĠT C +ĠVal ent +ĠRes cue +Ġby pass +am ount +Ġm ast +Ġprotect s +Ġart istic +Ġsomet ime +Ġsh oe +Ġshout ed +ific ant +et itive +ĠReg ister +ĠJ in +Ġconcent rated +ling ton +on ies +Ġgener ator +yr im +ĠAr men +Ġclear ing +id o +ĠT W +al ph +Ġlad ies +H ard +Ġdial og +Ġinput s +æ ľ +Ġpos es +Ġsl ots +ĠPrem ium +Ġle aks +Ġboss es +Ġ11 3 +c ourse +A cc +ĠNew ton +ĠAust ria +ĠM age +Ġte aches +ab ad +Ġwe ars +Ġc yl +Ġcur se +ĠS ales +ĠW ings +Ġp sy +Ġg aps +ĠIce land +ĠP interest +Ġland lord +Ġdefin itions +ĠK er +Ġsufficient ly +ĠP ence +ĠArch itect +Ġsur pass +Ġ11 4 +Ġsuper hero +ĠDise ase +Ġpri ests +ĠC ulture +Ġdefin itive +Ġsecret ly +ĠD ance +inst all +ch ief +ĠJess ica +W ould +Up dated +Ġlock er +ĠK ay +Ġmem orial +è ¦ +f at +Ġdis gu +Ġflav ors +ĠBase ball +ĠRes istance +Ġk icks +Ġen v +Ġteen agers +D ark +ĠC AR +Ġh alt +ĠL G +ĠGab riel +Ġfe ver +Ġs atur +Ġm all +Ġaffili ate +ĠS leep +ĠSpe cific +ĠV el +Ġj ar +ĠSac red +ĠEd wards +ĠA CL +Ġret ained +ĠG iant +Ġlim itation +in ces +Ġref usal +ĠT ale +ĠBut ler +Ġacc idents +ĠC SS +Ġimport ed +ĠCop y +Î ± +ER T +z el +Ġdiv isions +h ots +ĠAl b +ĠD S +Load er +W ashington +at isf +ĠCreat ive +\ . +ĠAut om +red ict +Ġrecept or +ĠCarl os +Met hod +ok a +Ġmal icious +Ġste pping +, [ +ĠD ad +Ġatt raction +ĠEffect s +ĠPir ate +ĠC er +ĠIndust ry +ĠR ud +Ġchar ter +Ġd ining +Ġins ists +Ġconfig ure +Ġ( # +ĠSim ple +ĠSc roll +UT C +17 5 +ĠK on +Ġmarket place +Ġ ãĤ +Ġref res +Ġg ates +er red +ĠP od +Ġbeh ave +Fr ank +n ode +Ġendors ed +he tt +as ive +ĠHom eland +Ġr ides +ĠLe ave +er ness +Ġflood ing +A FP +Ġris en +Ġcontin ually +Ġun anim +ĠCont ract +ĠP as +Ġgu ided +ĠCh ile +b d +Ġsu cc +pt ic +Ġcomm ittees +ĠL uther +ĠAny one +Ġs ab +12 4 +Ġp ixel +ĠB ak +ĠT ag +ĠBenn ett +En ter +sm all +ĠPresident ial +Ġp ul +Ġcontr ace +arch ive +Ġcoast al +ĠK ids +19 2 +âĢ ² +ick y +ING TON +Ġw olf +ĠSt alin +T ur +id get +am as +ĠUn less +Ġspons or +Ġmor ph +ĠCho ose +Ġrun ner +Ġun bel +Ġm ud +ĠMan a +Ġdub bed +Ġg odd +ure rs +wind ow +Ġrel ied +Ġcelebr ating +os c +Ġ13 5 +Ġlobb ying +Ġincom plete +Ġrestrict ion +Ġinc ap +it us +Ġexpect ation +ĠAp ollo +Ġint ens +Ġsyn c +G H +Ġmanip ulation +B Y +Ġspe ar +Ġbre asts +Ġvol can +il ia +M aterial +Ġform ats +ĠB ast +Ġparliament ary +Ġsn ake +Ġserv ants +ĠTr udeau +ĠGr im +ĠArab ic +ĠSC P +ĠBoy s +st ation +Ġprospect ive +ord e +in itialized +Ġb ored +AB LE +Ġaccess ed +Ġtax i +ĠShe ll +aid en +urs ed +in ates +ĠIns urance +ĠPet e +Sept ember +6 50 +Ġad ventures +ĠCo ver +Ġt ribute +Ġsk etch +Ġem power +Ġ Ø +ĠGl enn +ĠD aw += \" +ĠPolit ics +Ġgu ides +Ġd ioxide +ĠG ore +ĠBr ight +ĠS ierra +Ġval ued +c ond +Ġpo inter +Se lect +Ġrisk y +Ġabsor b +im ages +Ġref uses +Ġbon uses +__ _ +Ġh ilar +ĠF eatures +2 20 +ĠCollect or +F oot +Ġ19 64 +cul us +Ġd awn +Ġwork out +ĠL O +Ġphilosoph ical +ĠSand y +ĠYou th +Ġl iable +A f +bl ue +Ġovert urn +less ness +ĠTrib une +ĠIn g +Ġfact ories +Ġcat ches +Ġpr one +Ġmat rix +Ġlog in +Ġin acc +Ġex ert +s ys +Ġneed le +ĠQ ur +Ġnot ified +ould er +t x +Ġremind s +Ġpublisher s +Ġn ort +Ġg it +Ġfl ies +ĠEm ily +Ġflow ing +ĠAl ien +ĠStr ateg +Ġhard est +Ġmod ification +AP I +ĠM Y +Ġcr ashes +st airs +n umber +Ġur ging +ch annel +ĠFal con +Ġinhabit ants +Ġterr ifying +Ġutil ize +Ġban ner +Ġcig arettes +Ġsens es +ĠHol mes +Ġpract ition +ĠPhill ips +ott o +Ġcomp ile +Mod el +ĠK o +Ġ[ ] +Americ ans +ĠTer ms +Ġmed ications +ĠAn a +Ġfundament ally +ĠNot ice +Ġwe aker +Ġ 0000 +Ġgar lic +Ġout break +Ġeconom ist +ĠB irth +Ġobst acles +ar cer +ĠOr thodox +Ġplace bo +ĠC rew +asp berry +ĠAng els +Ġdis charge +Ġdestruct ive +11 7 +ĠR ising +Ġd airy +l ate +Ġcoll ision +ĠTig ers +ean or +ocument ed +ĠIn valid +Ġd ont +ĠL iter +ĠV a +Ġhyd rogen +Ġvari ants +ĠBrown s +Ġ19 65 +Ġind igenous +Ġtrad es +Ġremain der +Ġswe pt +ĠImp act +Ġred ist +Ġun int +grad uate +ãĥ ķ +ĠW ILL +ãģ® ç +ĠCrit ical +Ġf isher +Ġv icious +Ġrevers ed +Y ear +ĠS ox +Ġshoot ings +Ġfil ming +Ġtouchdown s +ai res +m el +Ġgrand father +Ġaffect ion +ing le +Ġover ly +Add itional +Ġsup reme +ĠGr ad +Ġsport ing +Ġmer cy +ĠBrook s +ount y +Ġperform s +Ġtight ly +Ġdem ons +Ġkill ings +Ġfact ion +ĠNov a +aut s +Ġund oubtedly +ar in +Ġunder way +ra k +Ġl iv +ĠReg ion +Ġbrief ing +s ers +cl oud +ĠM ik +us p +Ġpred iction +az or +Ġport able +ĠG and +Ġpresent ing +Ġ10 80 + » +ush i +ĠSp ark +there um +Ġjust ification +ĠN y +Ġcontract ors +ming ham +ĠSt yle +å ħ +ĠChron icles +ĠPict ure +Ġprov ing +Ġw ives +set t +Ġmole cules +ĠFair y +Ġconsist ing +Ġp ier +al one +in ition +Ġn ucle +j son +Ġg otta +Ġmob il +Ġver bal +ar ium +Ġmon ument +uck ed +Ġ25 6 +T ech +mine craft +ĠTr ack +Ġt ile +Ġcompat ibility +as is +Ġs add +Ġinstruct ed +ĠM ueller +Ġle thal +Ġhorm one +Ġor che +el se +Ġske let +Ġentert aining +Ġminim ize +ag ain +Ġunder go +Ġconst raints +Ġcig arette +ĠIslam ist +Ġtravel s +ĠPant hers +l ings +C are +Ġlaw suits +ur as +Ġcry st +Ġlow ered +Ġaer ial +Ġcomb inations +Ġha un +Ġch a +Ġv ine +Ġquant ities +Ġlink ing +b ank +Ġso y +B ill +ĠAngel a +Ġrecip ient +ĠProt est +Ġs ocket +Ġsolid arity +Ġâ Ĩ +m ill +Ġvar ies +ĠPak istani +Dr agon +Ġun e +Ġhor izon +³³³³ ³³³³ +Ġprov inces +Ġfrank ly +Ġenact ed +not es +[ ' +Ġ19 2 +ocr acy +Ġendorse ment +Ġover time +Tr ue +L ab +lic ted +ĠD NC +Ġbe ats +ĠJam ie +15 2 +ĠIN T +Cont act +Ġaccount ed +h ash +ĠPack ers +p ires +Ġles bian +Ġamend ments +Ġhop eful +ĠFin land +Ġspot light +Ġconfig ured +Ġtrou bled +Ġg aze +ĠCal gary +Ġrel iability +Ġins urg +sw er +b uy +ĠSk in +Ġp ixels +Ġhand gun +Ġpar as +Ġcateg or +ĠE L +ĠRe x +Ind eed +Ġkind a +Ġconj unction +ĠBry an +ĠMan ufact +y ang +Pl us +S QL +ish ment +Ġdom inate +Ġn ail +Ġo ath +Ġeru pt +ĠF ine +it bart +ĠCh ip +ĠAb d +ĠN am +Ġbuy er +Ġdiss ent +Le aks +Cont in +Ġr ider +ĠSome one +Ġill usion +c in +ĠBoe ing +Ġin adequ +ov ation +i ants +Ġreb uild +4 50 +ĠDest iny +S W +ĠT ill +H it +ia z +ĠBang l +acher s +ĠRe form +Ġse gments +Ġsystem atic +d c +ĠConserv atives +Ġport al +h or +ĠDragon bound +Ġdrag ged +om o +Ġthe e +ad vert +ĠRep orts +ĠE t +Ġbarrel s +Aug ust +Ġcompar isons +Ġhe x +Ġan throp +" [ +bor ough +ab i +Ġpict ured +play ing +ĠAdd ress +ĠMir ror +Sm ith +Ġt ires +ĠN PR +AA AA +Ġclass ification +ĠTh an +ĠH arm +ĠR A +Ġreject ion +min ation +Ġr anged +ĠF alls +D I +H ost +ãĤ ´ +ĠEx ample +list ed +th irds +Ġsaf egu +br and +Ġprob able +Can ada +IT ION +ĠQ aeda +Ġch ick +Ġimport s +h it +l oc +W W +Ġble w +Ġany time +Ġwh oles +ik ed +Ġcal culation +cre ate +ĠO ri +Ġupgr aded +Ġapp ar +ut ory +ĠM ol +B rit +ĠJ ong +IN AL +ĠStart ing +Ġd ice +urt le +Ġre lying +cl osure +Ġprof itable +Ġsl aughter +ĠMan ual +c aster +Ġ" $ +Ġfe ather +ĠSim ply +ie ves +Ġdeter ior +ĠPC I +Ġst amp +Ġfl aws +Ġsh ade +ham mer +Ġpass port +Ġcont ing +am el +Ġobser vers +Ġneg lect +ĠR B +ĠBrother hood +Ġskept ical +f amily +us k +Ġemotion ally +â Ļ +ĠBet a +ason able +id ity +ĠM ul +Ġkick ing +ĠC arm +oll ah +VERT IS +ĠAt hen +Ġlad der +ĠBul let +å £ +00 01 +ĠWild life +ĠM ask +ĠN an +R ev +Ġun acceptable +leg al +Ġcrowd ed +ag i +ĠC ox +j e +Ġmor ality +Ġfu els +Ġc ables +Ġman kind +ĠCarib bean +Ġanch or +Ġby te +ĠO ften +ĠO z +Ġcraft ed +Ġhistor ian +ĠW u +Ġtow ers +ĠCitiz ens +Ġhel m +Ġcred entials +Ġsing ular +ĠJes se +Ġtack les +Ġcont empt +Ġa fore +ĠSh adows +Ġn il +Ġur gent +app le +bl ood +Ġv on +Ġoff line +Ġbreat he +Ġj umps +Ġirre levant +ox ic +om al +import ant +J im +Ġgl oves +arm ing +dep th +Ġtal ents +ook ie +ĠS B +Ġpal m +uff s +est a +IG H +Ġcan on +ĠVer izon +ĠP le +Ġcou pled +vel t +Ġfundra ising +ĠGet ting +ĠD LC +Ġmathemat ical +ĠH S +ĠCard inals +te lling +Ġspons ors +Ġ Ï +ĠBull s +op tion +Ġprop ose +Ġmem orable +Ġembr aced +Ġdecl ining +He alth +ed a +Ġ} ; +Ġsp am +m ile +Ġpit cher +ĠE ight +Ġcar ing +ut ic +ro le +Ġair line +ernand ez +ĠAth let +Ġcert ification +ux e +rig er +Ġem pir +Ġsens ation +Ġdis m +Ġb olt +Ġev olve +H ouse +Ġconsult ation +ĠD uty +Ġtou ches +ĠN athan +Ġf aint +h ad +" ( +ĠCons umer +ĠExt reme +Ġ12 7 +ĠHer m +ĠSac rament +iz oph +Ġanx ious +ul ously +Ġsoc ially +ĠU TC +Ġsol ving +ĠLet ter +Hist ory +ed uc +Pr ice +) ); +Ġrel oad +am ic +Ġp ork +Ġdisc ourse +Ġt ournaments +ai ro +ĠK ur +ĠCost a +Ġviol ating +Ġinterf ere +Ġrecre ational +uff le +Ġspe eches +Ġneed ing +Ġremem bers +Ġcred ited +n ia +f ocused +amer a +Ġb ru +um bs +ĠCub an +Ġpreced ing +Ġnons ense +ac ial +Ġsmart phones +ĠSt ories +S ports +ĠEmer gency +oun cing +ef ined +Ġb er +Ġconsult ing +Ġm asters +he astern +." [ +ĠRun ning +Ġsus cept +ĠF eng +Americ a +pr ises +st itial +ĠWeek ly +ĠGreat er +mod ules +if ter +G raphics +ul er +Ġwho lly +Ġsupp ress +Ġconce aled +Ġhapp ily +Ġaccept s +ĠEn joy +Ġr ivers +ĠEx cept +2 25 +ĠN HS +ĠMc Connell +Ġp ussy +fer red +ut able +Ġatt ain +Ġ> = +Ġdepos its +roph ic +Ġnot orious +ĠSh aw +il itation +Ġepid emic +all ic +Ġsmall est +ov ich +Ġaccess ories +per ties +Ġsur plus +ĠMe ch +Ġamb ig +ĠImm igration +Ġch im +ev al +Ġpract icing +ĠMyster y +Ġdom ains +ĠSil icon +app s +Ġkilomet ers +e a +ĠSm ash +Ġwarrant y +Ġn ost +s il +re v +J on +ĠDub lin +Ġtast es +Ġb out +g reat +er ror +Ġsw itches +ĠB apt +D O +ok i +Ġsour ced +pro du +Ġattach ment +ĠIss ue +ĠQuest ion +Jo in +Ġf itted +Ġunlaw ful +^ ^ +ere k +Ġauthent ication +Ġst ole +Ġaccount ability +l abel +S earch +Ġal beit +atic an +fund ed +ĠAdd ing +ĠI Q +Ġsub mar +l it +a que +ĠLear ning +Ġint eger +M aster +ĠCh rom +Ġprem ier +O p +ĠLi u +Ġbl essed +ĠGl obe +ĠResp onse +Ġlegit im +ĠMer kel +Ġdispos al + ´ +Ġgau ge +pe at +Ġindu ced +Ġquestion able +arth y +ĠV it +ĠF eed +U ntil +U t +worth y +R Y +ĠH erald +ĠHam mer +Ġmed al +ĠR ivers +ĠH ack +Ġclar ify +Ġtrack ed +Ġautonom ous +Ġten ant +ĠQ atar +er ie +Ġgr im +ĠMon itor +Ġresist ant +ĠSpe c +ĠWell s +N AS +14 8 +Ġmin ers +iot ics +Ġmiss es +11 6 +g ian +g it +ĠE yes +p res +Ġgrad uated +Ġang el +Ġsyn chron +Ġefficient ly +Ġtrans mitted +H arry +Ġglob ally +EN CE +ĠMont ana +r aged +ĠPre vention +Ġp iss +ĠL l +Ġshe lf +ĠB JP +ĠTest ament +ĠL ate +ik er +ĠH app +ĠJul ian +h all +Ġsp ont +Ġshut down +Ġincons istent +Ġsubscrib ers +Ġske leton +ĠNe braska +Ġins pire +ĠV oid +F eed +Ġang les +ĠSpr ings +Ġbench mark +Ġvacc ines +izoph ren +se xual +uff ed +Ġsh ine +ĠK ath +Ġgest ure +ine a +Ġr ip +Ġopp ression +Ġcons cience +b t +ĠL um +Ġinc idence +ĠF a +w r +Ġmin eral +ĠSp urs +alk y +Ġth under +Ġop io +Be ing +ĠPal m +Ġwas ted +Ġl b +i aries +ĠIniti ative +Ġcur ric +Ġmark er +ĠMc L +Ġext ensions +ĠP v +ĠAr ms +Ġoffer ings +Ġdef enses +Ġvend or +Ġcontrad ict +ĠCol in +Ġredd it +Ġper ipher +12 2 +Ġs ins +E dit +IC T +So ft +ĠSh ah +Ġadministr ator +ĠT rip +Ġporn ography +Ġtu ition +in ence +ĠPro gress +Ġcat alog +Ġsu ite +Ġh ike +Ġreprodu ctive +eng ine +Ġd rought +ĠNo ah +Ġ2 30 +Ġd ude +Ġrelax ed +Ġpart ition +Ġparticip ant +Ġtel esc +Ġfe as +ĠF F +own er +Ġswe eping +Ġl enses +Ġmatch up +ĠRe pl +ourn als +Ġcred ible +Ġgrand mother +Ġther mal +Ġsubscrib ing +Ġident ities +col m +U CT +Ġreluct ant +us ers +ĠC ort +Ġassist ed +OS S +ATION S +IS H +Ġpharm aceutical +ic able +ad ian +ĠSon ic +ĠF ury +ĠM ong +A H +ĠPsych ology +Ġph osph +Ġtreat s +Ń Ķ +Ġstead ily +ĠHell o +Ġrel ates +Ġcl ue +Ex pl +a uth +Ġrev ision +Ġe ld +os ion +Ġbr on +14 4 +ri kes +Ġmin es +Ġblank et +ĠF ail +el ed +ĠIm agine +ĠPl anned +a ic +Re quest +M ad +ĠHor se +ĠEag le +Ġcap ac +15 7 +Ġl ing +ĠN ice +ĠP arenthood +min ster +og s +ens itive +Not hing +Ġcar n +F in +ĠP E +Ġr ifles +ĠL P +S and +Ġgui Active +Ġtour ist +C NN +Ġunve iled +Ġpredec essor +} { +u ber +Ġoff shore +Ġopt ical +ĠR ot +ĠPear l +et on +Ġst ared +Ġfart her +at ility +cont in +ĠG y +ĠF oster +ĠC oc +ri ents +Ġdesign ing +ĠEconom y +ON G +W omen +ĠN ancy +er ver +Ġmas cul +Ġcasual ties +Ġ2 25 +ĠS ullivan +ĠCh oice +Ġa ster +w s +Ġhot els +Ġconsider ations +Ġcou ch +ĠSt rip +ĠG n +Ġmanip ulate +l ied +Ġsynt hetic +Ġassault ed +Ġoff enses +ĠDra ke +Ġim pe +Oct ober +ĠHer itage +h l +ĠBl air +Un like +Ġg rief +Ġ4 50 +Ġopt ed +Ġresign ation +il o +Ġver se +ĠT omb +Ġu pt +Ġa ired +ĠH ook +ĠML B +Ġassum es +out ed +ĠV ers +Ġinfer ior +Ġbund le +ĠD NS +ograp her +Ġmult ip +ĠSoul s +Ġillust rated +Ġtact ic +Ġdress ing +Ġdu o +Con f +Ġrel ent +Ġc ant +Ġscar ce +Ġcand y +ĠC F +Ġaffili ated +Ġspr int +yl an +ĠGarc ia +Ġj unk +Pr int +ex ec +C rit +Ġport rait +ir ies +ĠOF F +Ġdisp utes +W R +L ove +ãģ Ħ +ĠRe yn +Ġh ipp +op ath +Ġflo ors +ĠFe el +Ġwor ries +Ġsett lements +ĠP os +Ġmos que +Ġfin als +Ġcr ushed +ĠPro bably +ĠB ot +ĠM ans +ĠPer iod +Ġsovere ignty +Ġsell er +Ġap ost +Ġam ateur +Ġd orm +Ġconsum ing +Ġarm our +ĠRo ose +Ġint ensive +Ġelim inating +ĠSun ni +ĠAle ppo +j in +Ġadv ise +p al +ĠH alo +Ġdes cent +Ġsimpl er +Ġbo oth +ST R +L ater +ĠC ave +== = +Ġm ol +Ġf ist +Ġshot gun +su pp +Ġrob bery +E ffect +Ġobsc ure +ĠProf essional +Ġemb assy +Ġmilit ant +Ġinc arcer +Ġgener ates +Ġlaun ches +Ġadministr ators +Ġsh aft +Ġcirc ular +Ġfresh man +ĠW es +ĠJo el +ĠD rew +ĠDun can +ĠApp arently +s ight +ĠIntern al +ĠInd ividual +ĠF E +Ġb ore +ĠM t +Ġbroad ly +ĠO ptions +ount ain +ip es +ĠV ideos +20 4 +Ġh ills +Ġsim ulation +Ġdisappoint ment +it an +ĠLabor atory +Ġup ward +Ġbound ary +Ġdark er +h art +Ġdomin ance +C ong +ĠOr acle +ĠL ords +Ġscholars hip +ĠVin cent +ed e +ĠR ah +Ġencour ages +ro v +Ġqu o +Ġprem ise +ĠCris is +ĠHol ocaust +Ġrhyth m +Ġmet ric +cl ub +Ġtransport ed +Ġn od +ĠP ist +Ġancest ors +ĠFred er +th umbnails +ĠC E +ON D +Ph il +ven ge +ĠProduct s +cast le +Ġqual ifying +ĠK aren +VERTIS EMENT +Ġmight y +Ġexplan ations +Ġfix ing +D i +Ġdecl aring +Ġanonym ity +Ġju ven +ĠN ord +ĠDo om +ĠAct ually +O k +ph is +ĠDes ert +Ġ11 6 +I K +ĠF M +Ġinc omes +V EL +ok ers +Ġpe cul +Ġlight weight +g ue +Ġacc ent +Ġincre ment +ĠCh an +Ġcompl aining +ĠB aghd +Ġmidfield er +Ġover haul +Pro cess +ĠH ollow +ĠTit ans +Sm all +man uel +ĠUn ity +ĠEv ents +S ty +Ġdispro portion +n esty +en es +ĠC od +Ġdemonstr ations +ĠCrim son +ĠO H +Ġen rolled +Ġc el +ĠBre tt +Ġa ide +Ġhe els +Ġbroad band +Ġmark ing +Ġw izard +ĠN J +ĠChief s +Ġingred ient +Ġd ug +ĠSh ut +urch ase +end or +Ġfar mer +ĠGold man +12 9 +15 5 +Or der +Ġl ion +i ably +Ġst ain +ar ray +ilit ary +ĠFA Q +Ġexpl oded +ĠMcC arthy +ĠT weet +ĠG reens +ek ing +l n +ens en +Ġmotor cycle +Ġpartic le +Ġch olesterol +B ron +Ġst air +Ġox id +Ġdes irable +ib les +Ġthe or +for cing +Ġpromot ional +ov o +b oot +ĠBon us +raw ling +Ġshort age +ĠP sy +Ġrecru ited +Ġinf ants +Ġtest osterone +Ġded uct +Ġdistinct ive +Ġfirm ware +bu ilt +14 5 +Ġexpl ored +Ġfact ions +Ġv ide +Ġtatt oo +Ġfinan cially +Ġfat igue +Ġproceed ing +const itutional +Ġmis er +Ġch airs +gg ing +ipp le +Ġd ent +Ġdis reg +ç Ķ +st ant +ll o +b ps +aken ing +Ġab normal +ĠE RA +å£ « +ĠH BO +ĠM AR +Ġcon cess +Ġserv ant +Ġas pir +l av +ĠPan el +am o +Ġprec ip +Ġrecord ings +Ġproceed ed +Ġcol ony +ĠT ang +ab lo +Ġstri pped +Le ft +to o +Ġpot atoes +Ġfin est +% ). +Ġc rap +ĠZ ach +ab ases +ĠG oth +Ġbillion aire +w olf +Ġsan ction +S K +Ġlog ged +P o +ey ed +un al +Ġcr icket +Ġarm ies +Ġunc overed +Cl oud +ó n +Ġreb ounds +Ġm es +O per +P ac +Ġnation ally +Ġinsert ed +p ict +Ġgovern ance +Ð ¸ +Ġprivile ges +G ET +Ġfavor ites +im ity +Ġlo ver +the m +em pl +Ġgorge ous +An n +Ġsl ipped +Ġve to +B ob +Ġsl im +u cc +ĠF ame +udden ly +Ġden ies +ĠM aur +Ġdist ances +Ġw anna +t ar +ĠS ER +Ġâ Ī +Ġle mon +at hetic +Ġlit eral +Ġdistingu ished +Ġansw ering +G I +Ġrelig ions +ĠPhil os +ĠL ay +Ġcomp os +ire ments +ĠK os +ine z +roll ing +Ġyoung est +and ise +ĠB orn +Ġalt ar +am ina +ĠB oot +v oc +Ġdig ging +Ġpress ures +Ġl en +26 4 +Ġassass ination +ĠBir mingham +ĠMy th +Ġsovere ign +ĠArt ist +ĠPhot ograph +Ġdep icted +Ġdisp ens +orth y +Ġamb ul +int eg +ĠC ele +ĠTib et +Ġhier archy +Ġc u +Ġpre season +ĠPet erson +Ġcol ours +Ġworry ing +Ġback ers +ĠPal mer +ĠÎ ¼ +Ġcontribut or +Ġhear ings +Ġur ine +Ġ Ù +ourge ois +Sim ilar +ĠZ immer +s omething +ĠUS C +Ġstrength s +ĠF I +Ġlog ging +As ked +ĠTh ai +in qu +ĠW alt +Ġcrew s +it ism +3 01 +Ġshar ply +um ed +Ġred irect +r ators +In f +ĠWe apons +Ġte asp +19 99 +L ive +ĠEs pecially +ĠS ter +ĠVeter ans +Ġint ro +other apy +Ġmal ware +Ġbre eding +Ġmole cular +ĠR oute +ĠCom ment +oc hem +Ġa in +Se ason +Ġlineback er +Ä « +ĠEconom ics +es ar +ĠL ives +ĠEm ma +Ġk in +ĠTer rit +Ġpl anted +ot on +ĠBut ter +ĠSp ons +P ER +Ġdun geon +Ġsymb olic +Ġfil med +Ġdi ets +Ġconclud es +Ġcertain ty +ĠForm at +Ġstr angers +form at +ĠPh ase +Ġcop ied +Ġmet res +ld a +ĠUs ers +Ġdeliber ate +Ġwas hed +ĠL ance +im ation +Ġimpro per +ĠGen esis +ick r +ĠK ush +Ġreal ise +Ġembarrass ing +alk ing +b ucks +Ġver ified +Ġout line +year s +ĠIn come +20 2 +Ġz ombies +F inal +ĠMill enn +Ġmod ifications +ĠV ision +ĠM oses +ver b +iter ranean +ĠJ et +Ġnav al +ĠA gg +Ġur l +Ġvict ories +Ġnon etheless +Ġinj ust +ĠF act +ç ļ +Ġins ufficient +re view +face book +Ġnegoti ating +Ġguarant ees +im en +uten berg +Ġg ambling +Ġcon gr +Load ing +Ġnever theless +Ġpres idents +ĠIndust rial +Ġ11 8 +Ġp oured +ĠT ory +Ġ17 5 +Ġ: = +Sc ott +ange red +T ok +Ġorgan izers +M at +ĠG rowth +Ġad ul +Ġens ures +Ġ11 7 +é¾į å +Ġmass acre +Ġgr ades +be fore +AD VERTISEMENT +ĠSl ow +ĠM MA +âĢĶ " +ĠV atican +Q aeda +Ġo we +66 66 +ĠS orry +ĠGr ass +Ġbackground s +Ġexha usted +Ġcl an +Ġcomprom ised +ĠE lf +ĠIsa ac +ens on +In vest +IF A +Ġinterrupt ed +ãĥī ãĥ© +Ġtw isted +ĠDrag ons +M ode +ĠK remlin +Ġfert il +he res +ph an +ĠN ode +f ed +ĠOr c +Ġunw illing +C ent +Ġprior it +Ġgrad uates +Ġsubject ive +Ġiss uing +ĠL t +Ġview er +Ġw oke +Th us +bro ok +Ġdep ressed +Ġbr acket +ĠG or +ĠFight ing +Ġstri ker +Rep ort +ĠPortug al +Ġne o +w ed +19 9 +Ġflee ing +sh adow +ident ified +US E +Ste am +Ġstret ched +Ġrevel ations +art ed +ĠD w +Ġalign ment +est on +ĠJ ared +S ep +Ġblog s +up date +g om +r isk +Ġcl ash +ĠH our +Ġrun time +Ġunw anted +Ġsc am +Ġr ack +Ġen light +on est +ĠF err +Ġconv ictions +Ġp iano +Ġcirc ulation +ĠW elcome +Ġback lash +ĠW ade +Ġrece ivers +ot ive +J eff +Ġnetwork ing +ĠPre p +ĠExpl orer +Ġlect ure +Ġupload ed +ĠMe at +B LE +ĠNaz is +ĠSy nd +st ud +ro ots +ri ans +Ġportray ed +Ġ ?? +ĠBudd ha +s un +Rober t +ĠCom plex +Ġover see +Ġste alth +T itle +ĠJ obs +ĠK um +Ġappreci ation +ĠM OD +Ġbas ics +Ġcl ips +Ġnurs ing +Ġpropos ition +Ġreal ised +ĠNY C +Ġall ocated +ri um +ar an +ĠPro duction +ĠV ote +Ġsm ugg +Ġhun ter +az er +ĠCh anges +Ġfl uct +y on +Ar ray +Ġk its +W ater +Ġuncom mon +Ġrest ing +ell s +w ould +Ġpurs ued +Ġassert ion +omet own +ĠMos ul +ĠPl atform +io let +Ġshare holders +Ġtra ils +P ay +ĠEn forcement +ty pes +ĠAn onymous +Ġsatisf ying +il ogy +Ġ( ' +w ave +c ity +Ste ve +Ġconfront ation +ĠE ld +C apt +ah an +ht m +ĠC trl +ON S +2 30 +if a +hold ing +Ġdelic ate +Ġj aw +ĠGo ing +or um +S al +Ġd ull +ĠB eth +Ġpr isons +Ġe go +ĠEl sa +avor ite +ĠG ang +ĠN uclear +Ġsp ider +ats u +Ġsam pling +Ġabsor bed +ĠPh arm +iet h +Ġbuck et +ĠRec omm +O F +ĠF actory +AN CE +Ġb acter +H as +ĠObs erv +12 1 +Ġprem iere +De velop +Ġcur rencies +C ast +Ġaccompany ing +ĠNash ville +Ġfat ty +ĠBre nd +Ġloc ks +Ġcent ered +ĠU T +augh s +or ie +ĠAff ordable +v ance +D L +em et +Ġthr one +ĠBlu etooth +Ġn aming +if ts +AD E +Ġcorrect ed +Ġprompt ly +ĠST R +Ġgen ome +Ġcop e +Ġval ley +Ġround ed +ĠK end +al ion +p ers +Ġtour ism +Ġst ark +v l +Ġblow ing +ĠSche dule +st d +Ġunh appy +Ġlit igation +ced es +Ġand roid +Ġinteg ral +ere rs +ud ed +t ax +Ġre iter +ĠMot ors +oci ated +Ġwond ers +ĠAp ost +uck ing +ĠRoose velt +f ram +Ġyield s +Ġconstit utes +aw k +Int erest +Ġinter im +Ġbreak through +ĠC her +Ġpro sec +ĠD j +ĠM T +Res p +ĠP T +Ġs perm +ed it +B T +Lin ux +count ry +le ague +Ġd ick +Ġo ct +Ġinsert ing +Ġsc ra +ĠBrew ing +Ġ19 66 +Ġrun ners +Ġpl un +id y +ĠD ian +Ġdys function +Ġex clusion +Ġdis gr +Ġincorpor ate +Ġrecon c +Ġnom inated +ĠAr cher +d raw +achel or +Ġwrit ings +Ġshall ow +Ġh ast +ĠB MW +ĠR S +Ġth igh +Ġ19 63 +Ġl amb +Ġfav ored +ag le +Ġcool er +ĠH ours +ĠG U +ĠOrig in +Ġglim pse +---------------- ---- +L im +Ġche ek +Ġj ealous +- ' +Ġhar ness +ĠPo ison +Ġdis abilities +ne apolis +Ġout look +Ġnot ify +ĠIndian apolis +Ġab rupt +ns ic +Ġenc rypted +Ġfor fe +reat h +Ġr abb +Ġfound ations +Ġcompl iment +ĠInter view +ĠS we +Ġad olesc +Ġmon itors +ĠSacrament o +Ġtime ly +Ġcontem pl +Ġposition ed +Ġpost ers +ph ies +iov ascular +v oid +ĠFif th +Ġinvestig ative +OU N +Ġinteg rate +ĠIN C +ish a +ibl ings +ĠRe quest +ĠRodrig uez +Ġsl ides +ĠD X +Ġfemin ism +Ġdat as +Ġb end +ir us +ĠNig eria +F ox +Ch ange +Ġair plane +ĠLad en +Ġpublic ity +ixt y +Ġcommit ments +Ġaggreg ate +Ġdisplay ing +ĠAr row +Ġ12 2 +Ġrespect s +and roid +s ix +ĠSh a +Ġrest oration +) \ +W S +oy s +Ġillust rate +with out +12 6 +ĠâĶ Ĥ +Ġpick up +n els +Ġ .... +f ood +ĠF en +) ? +Ġphenomen a +Ġcompan ions +ĠW rite +Ġsp ill +Ġbr idges +ĠUp dated +ĠF o +Ġinsect s +ASH INGTON +Ġsc are +il tr +ĠZh ang +Ġsever ity +Ġind ul +14 9 +ĠCo ffee +Ġnorm s +Ġp ulse +ĠF T +Ġhorr ific +ĠDest roy +ĠJ SON +Ġo live +Ġdiscuss es +R est +E lect +ĠW inn +ĠSurv iv +ĠH ait +S ure +op ed +Ġro oted +ĠS ke +ĠBron ze +Ġl ol +Def ault +Ġcommod ity +red ited +Ġliber tarian +Ġforb idden +Ġgr an +à ¨ +Ġl ag +en z +dri ve +Ġmathemat ics +Ġw ires +Ġcrit ically +Ġcarb ohyd +ĠChance llor +ĠEd die +Ġban ning +ĠF ri +Ġcompl ications +et ric +ĠBangl adesh +Ġband width +St op +ĠOrig inally +Ġhalf way +yn asty +sh ine +Ġt ales +rit ies +av ier +Ġspin ning +ĠWH O +Ġneighbour hood +b ach +Ġcommer ce +ĠS le +B U +Ġentreprene ur +Ġpecul iar +ĠCom ments +f re +3 20 +IC S +Ġimag ery +ĠCan on +ĠElect ronic +sh ort +( ( +D ig +Ġcomm em +u ced +Ġincl ined +ĠSum mon +Ġcl iff +ĠMed iterranean +Ġpo etry +Ġprosper ity +ĠRe ce +Ġp ills +m ember +Ġfin ale +un c +ĠG ig +ä ½ +Ġl od +Ġback ward +- + +ĠFor ward +Ġth ri +s ure +Ġso ap +ĠF X +R ES +ĠSe xual +oul os +Ġfool ish +Ġright eous +Ġco ff +terror ism +ust ain +ot er +Ġab uses +ne xt +Ġab usive +Ġthere after +Ġprohib ition +ĠS UP +Ġd ip +Ġr ipped +Ġinher ited +Ġb ats +st ru +G T +Ġflaw ed +ph abet +Ġf og +do ors +Ġim aging +Ġdig its +ĠHung ary +Ġar rog +Ġteach ings +Ġprotocol s +ĠB anks +à ¸ +p ound +ĠC urt +." ) +. / +Ġex emption +end ix +ĠM ull +Ġimpro ves +ĠG amer +d imensional +I con +ĠMarg aret +St atus +d ates +Ġint ends +Ġdep ict +Ġpark ed +J oe +ĠMar ines +chn ology +! ). +Ġjud ged +Ġwe ights +R ay +Ġapart ments +he ster +Ġrein force +Ġoff ender +occ up +Ġs ore +e pt +ĠPH P +ĠB row +Ġauthor ization +ĠR isk +ĠDel aware +ĠQ U +Ġnot ifications +Ġsun light +Ġex clude +d at +Ġm esh +ĠSud an +Ġbelong ed +Ġsub way +Ġno on +ĠInter ior +ol ics +ĠL akers +Ġc oding +Dis claimer +Cal if +O ld +Ġdis l +???? ? +Ġconfir ms +Ġrecruit ment +Ġhom icide +Cons ider +ĠJeff rey +ft y +} ; +Ġobject ion +do ing +ĠLe o +W ant +Ġgl ow +ĠClar ke +ĠNorm an +Ġver ification +Ġpack et +ĠForm ula +Ġpl ag +es ville +Ġshout ing +Ġo v +ĠR EC +ĠB ub +Ġn inth +Ġener g +Ġvalid ity +Ġup s +j ack +Ġneighbor ing +ĠN ec +ew orks +ĠH ab +are z +Ġsp ine +Ġevent ual +ĠLe aders +ĠC arn +Ġprob ation +Ġrom ance +ms g +ĠMechan ical +ER Y +R ock +Ġpart isan +N ode +ass ets +min ent +Ġforeign ers +Ġtest ify +ĠUs ually +l ords +ĠG ren +ĠPow ell +BI L +Ġs r +Ġadd ict +Ġshell s +Ġs igh +ĠY ale +tern ity +Ġ7 50 +E U +ĠR ifle +Ġpat ron +em a +ĠB annon +an ity +Ġtrop ical +ĠV II +c ross +Every thing +ĠIS O +Ġhum ble +ass ing +ĠF IG +Ġupd ating +ys on +Ġcal cium +Ġcompet ent +Ġste ering +Pro t +ĠS Y +ĠFin als +ĠR ug +15 9 +13 7 +ĠG olf +Ġ12 6 +Ġaccommod ation +ĠHug hes +Ġaest hetic +art isan +ĠTw ilight +Ġpr ince +ĠAgric ulture +ĠDis co +Ġpreced ent +Ġtyp ing +author ized +O ption +ĠA ub +l ishes +ach t +m ag +P eter +ĠU FO +mont on +ĠL ith +Ġa rom +Ġsec uring +Ġconf ined +priv ate +Ġsw ords +Ġmark ers +Ġmetab olic +se lect +ĠCur se +ĠO t +g ressive +Ġinc umb +ĠS aga +Ġpr iced +Ġclear ance +Cont ent +Ġdr illing +Ġnot ices +Ġb ourgeois +Ġv est +Ġcook ie +ĠGuard ians +ry s +in yl +Ġ12 4 +Ġpl ausible +on gh +ĠOd in +Ġconcept ion +ĠY uk +ĠBaghd ad +ĠFl ag +Aust ral +ĠI BM +Ġintern ationally +ĠWiki Leaks +I ED +Ġc yn +Ġcho oses +ĠP ill +Ġcomb ining +Ġrad i +ĠMoh ammed +def ense +atch ing +Sub ject +ic iency +Fr ame +Ġ{ " +Ġche ss +Ġtim er +19 0 +Ġt in +Ġord inance +emet ery +Ġacc using +Ġnotice able +Ġcent res +Ġl id +ĠM ills +img ur +Ġz oom +erg ic +Ġcomp ression +pr im +f ind +Ġsur g +Ġp and +ĠK ee +ĠCh ad +cell ence +oy le +Ġsocial ism +ĠT ravis +ĠM Hz +Ġgu ild +ALL Y +ĠSub scribe +ĠRel ated +Ġoccur rence +itch ing +Ġfict ional +Ġcr ush +ĠE A +c od +m ix +ĠTri ple +Ġretrie ve +Ġstimul us +Ġpsych iat +ĠDo or +Ġhomosexual ity +Ġelement ary +Ġcell ular +id ian +ĠL aun +Ġintrig uing +Ġfo am +ĠB ass +id i +its u +Ġass ure +Ġcongr at +Ġbusiness man +ĠBo ost +cl ose +Ġl ied +Ġsc iences +ĠO mega +ĠG raphics +Ġ< = +sp oken +Ġconnect ivity +S aturday +ĠAven gers +Ġto ggle +Ġank le +Ġnational ist +mod el +ĠP ool +ophob ia +V ar +ĠM ons +ator ies +Ġaggress ively +C lear +For ge +act ers +Ġhed ge +Ġpip es +Ġbl unt +Ġs q +Ġremote ly +W ed +as ers +Ġref riger +Ġt iles +Ġresc ued +Ġcompr ised +ins ky +Ġman if +avan augh +Ġprol ifer +Ġal igned +x ml +Ġtri v +Ġcoord ination +ĠP ER +ĠQu ote +13 4 +b f +ĠS aw +Ġtermin ation +Ġ19 0 +Ġadd itions +Ġtri o +Ġproject ions +Ġpositive ly +Ġin clusive +Ġmem br +19 90 +old er +Ġpract iced +ink le +Ar ch +Ġstar ters +ari us +Ġinter mediate +ĠBen ef +ĠK iller +Ġinter ventions +ĠK il +ĠF lying +In v +Ġprem ature +Ġpsych iatric +Ġind ie +Ġcoll ar +ĠRain bow +af i +Ġdis ruption +ĠFO X +cast ing +Ġmis dem +c ro +Ġw ipe +ard on +Ġb ast +ĠTom my +ĠRepresent ative +Ġbell y +ĠP O +ĠBre itbart +13 2 +Ġmess aging +Sh ould +Ref erences +ĠG RE +ist ical +L P +ĠC av +ĠC razy +Ġintu itive +ke eping +ĠM oss +Ġdiscont in +ĠMod ule +Ġun related +ĠPract ice +ĠTrans port +Ġstatist ically +orn s +Ġs ized +p u +Ġca f +ĠWorld s +ĠRod gers +ĠL un +ĠCom ic +l iving +Ġc ared +Ġclim bed +) { +Ġconsist ed +Ġmed ieval +fol k +Ġh acked +Ġd ire +ĠHerm ione +Ġt ended +ce ans +D aniel +w ent +Ġlegisl ators +Ġred es +g ames +Ġg n +am iliar +Ġ+ + +gg y +th reat +Ġmag net +Ġper ceive +Ġz ip +Ġindict ment +Ġcrit ique +g ard +ĠSaf e +ĠC ream +Ġad vent +ob a +Ġv owed +ous ands +Ġsk i +Ġabort ions +u art +Ġstun ned +Ġadv ancing +Ġlack ed +Ġ\ " +Ġsch izophren +Ġeleg ant +Ġconf erences +Ġcance led +ĠHud son +ĠHop efully +Ġtr ump +Ġfrequ encies +Ġmet eor +ĠJun ior +ĠFle et +ĠMal colm +ĠT ools +Ġ ........ +Ġh obby +ĠEurope ans +Ġ15 00 +ĠInt o +Ġs way +ĠApp ro +ĠCom pl +Comm unity +Ġt ide +ĠSum mit +ä » +Ġinter vals +ĠE ther +Ġhabit at +ĠSteven s +lish ing +ĠDom ain +Ġtrig gers +Ġch asing +Ġchar m +ĠFl ower +it ored +Ġbless ing +Ġtext ures +F ive +Ġliqu or +R P +F IN +Ġ19 62 +C AR +Un known +Ġres il +ĠL ily +Ġabund ance +Ġpredict able +r ar +Ġbull shit +le en +che t +M or +M uch +ä ¹ +Ġemphas ized +Ġcr ust +Ġprim itive +Ġenjoy able +ĠPict ures +Ġteam mate +pl er +ĠT ol +ĠK ane +Ġsummon ed +th y +ram a +ĠH onda +Ġreal izing +Ġquick er +Ġconcent rate +cle ar +Ġ2 10 +ĠErd ogan +ar is +Ġrespond s +ĠB I +Ġelig ibility +Ġpus hes +ĠId aho +Ġagg rav +Ġru ins +ur ations +Ġb ans +Ġan at +sh are +Ġgr ind +h in +um en +Ġut ilities +ĠYan kees +Ġdat abases +ĠD D +Ġdispl aced +Ġdepend encies +Ġstim ulation +h un +h ouses +ĠP retty +ĠRaven s +ĠTOD AY +Ġassoci ates +Ġthe rape +cl ed +Ġde er +Ġrep airs +rent ice +Ġrecept ors +Ġrem ed +ĠC e +Ġmar riages +Ġball ots +ĠSold ier +Ġhilar ious +op l +13 8 +Ġinherent ly +Ġignor ant +Ġb ounce +ĠE aster +REL ATED +ĠCur rency +E V +ãĥ ŀ +ĠLe ad +Ġdece ased +B rien +ĠMus k +J S +Ġmer ge +heart ed +c reat +m itt +m und +ĠâĢ ĭ +ĠB ag +Ġproject ion +Ġj ava +ĠStand ards +ĠLeon ard +Ġcoc onut +ĠPop ulation +Ġtra ject +Ġimp ly +Ġcur iosity +ĠD B +ĠF resh +ĠP or +Ġheav ier +ne ys +gom ery +Ġdes erved +Ġphr ases +ĠG C +Ġye ast +d esc +De ath +Ġreb oot +Ġmet adata +IC AL +Ġrep ay +ĠInd ependence +Ġsubur ban +ical s +Ġat op +Ġall ocation +gener ation +ĠG ram +Ġmoist ure +Ġp ine +ĠLiber als +Ġa ides +Ġund erest +ĠBer ry +Ġcere mon +3 70 +ast rous +ĠPir ates +Ġt ense +ĠIndust ries +ĠApp eals +ĠN ear +Ġè£ı ç +Ġlo vers +ĠC AP +ĠC raw +Ġg iants +Ġeffic acy +E lement +ĠBeh avior +ĠToy ota +Ġint est +P riv +A I +Ġmaneu ver +Ġperfect ion +Ġb ang +p aper +r ill +Ge orge +b order +in ters +ĠS eth +Ġcl ues +ĠLe vi +ĠRe venue +14 7 +Ġv apor +Ġfortun ate +Ġthreat ens +Ġve t +Ġdepend ency +ers ed +art icle +ĠBl izzard +Ġch lor +Ġmin us +ĠB ills +Ġcryptoc urrency +Ġmetabol ism +ter ing +Ġp estic +step s +ĠTre asure +ract ed +ĠConst ant +Ġtem p +13 9 +ĠDet ective +ur ally +Ġrecover ing +Ġcort ex +Ġ14 4 +cl osed +Ġprejud ice +aun ted +Ġstorm s +ĠN OW +Ġmach inery +Add ress +Ġcompe lled +27 0 +Ġdesp air +b ane +Ġveget able +Ġbed s +Lear n +Ġcolor ful +Ġsp ike +Ġmarg ins +Ġsymp athy +Ġworks hop +ĠC BC +S at +Ġburn s +ĠG ender +Ġ12 9 +ĠC able +Ġdeb ts +ĠThe resa +Ġreflect ing +Ġa irst +Ġr im +ram id +Ġweakness es +W rit +ogg le +t i +ĠCh arge +Ġwe ighed +Ġ( . +Ġl aughter +Ġrou ter +ĠDemocr acy +D ear +Ġhas ht +Ġd y +Ġhint s +run ning +Ġfin ishes +ar us +M ass +res ult +asc us +Ġv intage +Ġcon qu +Ġwild ly +ac ist +Ġl ingu +Ġprot agonist +st rom +te enth +ĠSol o +m ac +f illed +Ġre nown +it ives +Ġmot ive +ĠAnt ar +ĠM ann +ĠAd just +Ġrock ets +Ġtrou bling +e i +Ġorgan isms +ass is +Christ ian +Ġ14 5 +ĠH ass +Ġsw all +Ġw ax +ĠSurv ival +V S +ĠM urd +v d +stand ard +Ġdrag ons +Ġacceler ation +r ational +f inal +Ġp aired +ĠE thereum +Ġinterf aces +Ġres ent +Ġartif acts +Å « +are l +Ġcompet itor +ĠNich olas +ĠSur face +c pp +ĠT ot +Ġeconom ically +Ġorgan ised +Ġen forced +in ho +Ġvar ieties +Ġab dom +ĠBa iley +id av +ĠSal v +p aid +Ġalt itude +ess ert +ĠG utenberg +are a +op oulos +Ġprofess ors +igg s +ĠF ate +he y +Ġ3 000 +D ist +Ġtw ins +c ill +ĠM aps +Ġtra ps +Ġwe ed +ĠK iss +Ġy oga +Ġrecip ients +ĠWest minster +Ġpool s +ĠWal mart +18 8 +ĠSchool s +att ack +ĠAR M +par agraph +W arning +j l +Ġself ish +anche z +ĠHe ights +F re +ĠS oph +Ġ -------------------------------- +t ml +33 3 +Ġraid s +Ġsatell ites +KE Y +Ġlast s +Ñ Ĥ +In s +ĠD ame +Ġunp redict +// / +gh ai +Ġart illery +Ġcru ise +Ġg el +ĠCabin et +Ġbl ows +ĠE sp +Ġprox imity +ot he +ĠSk ills +ĠU pper +ob o +ĠN DP +Ġenjoy s +Ġrepe ating +ĠConst ruction +ĠQuest ions +H illary +Ġu int +Ġprocess ors +ĠGib son +ĠMult iple +q a +ĠB om +ĠM iles +vent ional +Ġhur ts +s kin +ĠA IDS +Ġadvis ers +ĠR oot +Ġmethod ology +ĠD ale +Ġdet on +ĠKnow ledge +sequ ently +Ġ12 1 +Ġconnect s +C y +ĠD anger +Ġcontribut ors +ĠB ent +Ġbr ass +ĠGun s +int o +ĠFort une +Ġbro ker +bal ance +Ġlength s +Ġv ic +Ġaver aging +Ġappropri ately +ĠCamer a +Ġsand wich +ĠCD C +Ġcoord inate +Ġnav ig +Ġgood ness +l aim +Ġbra ke +Ġextrem ist +ĠW ake +ĠM end +ĠT iny +ĠC OL +ĠR F +ĠD ual +ĠW ine +C ase +Ġref ined +Ġl amp +L ead +Ġb apt +ĠCar b +ĠS add +ĠMin neapolis +PD F +Ear ly +ĠH idden +I ts +ĠT IME +Ġp ap +Ġcommission ed +ĠF ew +ĠCol ts +ĠB ren +Ġbot hered +Ġlike wise +Ex per +ĠSch w +c ry +n n +ĠM itch +im on +M G +b m +UM P +r ays +Ġregist ry +Ġ2 70 +ach ine +re lla +ant ing +00 000 +Ġru ined +sp ot +Ġt a +Ġmaxim ize +Ġincon ven +D ead +H uman +En abled +ĠMar ie +Ġch ill +ĠParad ise +Ġstar ring +ĠLat ino +ĠProt ocol +ĠE VER +Ġsuppl iers +m essage +ĠBro ck +Ġser um +âĸĪâĸĪ âĸĪâĸĪ +Ġen comp +Ġamb ition +ues e +Ġar rows +And rew +Ġanten na +Ġ19 61 +ĠB ark +Ġb ool +ãĤ ª +ĠSt orage +Ġrail way +Ġtoug her +ĠC ad +Ġwas hing +P y +' ] +em bed +ĠMem phis +ack le +Ġfam ously +ĠF ortunately +ov ies +Ġmind set +Ġsne ak +ĠD h +RA W +ĠSim pson +Ġliv est +Ġland mark +Ġc ement +L ow +Ġthr illed +ĠCour se +in el +Ġch uck +id ate +gl obal +Ġwh it +Ġ � +ad ays +s ki +ĠS V +Ġvir uses +30 6 +ĠResp ons +Ġthe aters +ĠBr anch +ĠGene va +ĠM K +Ġunbel iev +Ġcommun ist +Orig inal +ĠRe ceived +ĠTrans fer +ĠAr g +In put +ĠStr ategy +Ġpal ace +the ning +D ri +Ġsent encing +umbn ail +Ġp ins +re cy +Ġs iblings +Get ting +ĠB U +ĠNorth west +Ġprolong ed +ĠSak ura +C omb +ĠB our +Ġinadequ ate +ĠK ash +Ġus ername +ĠImpro ve +Ġbatt ling +ĠM AC +Ġcurric ulum +Ġs oda +ĠC annon +Ġsens ible +sp ons +De cember +Ġw icked +ĠP engu +Ġdict ators +ĠHe arts +og yn +Ġsimilar ities +ĠSt ats +Ġh ollow +it ations +": [ +Ġh over +ĠList en +s ch +S und +Ġc ad +ĠPar ks +Ġl ur +Ġhy pe +ĠL em +N AME +is ure +Fr iday +Ġshoot s +Ġclos es +Ġd b +ĠR idge +ĠDiff erent +Ġrepl ies +ĠBroad way +op ers +Ġint oler +ĠZe us +akes pe +Ġpropri etary +Ġrequest ing +Ġcontro llers +ĠM IN +im edia +be cca +Ġexp ans +Ġoil s +B ot +ĠCh and +Ġpr inter +Ġto pped +ĠP OL +ĠEar lier +S ocial +av in +Ġdecre ases +ĠSe b +Ġspecific ations +ĠBl ast +ĠK urt +Ġfre el +B rown +Ġdil ig +ro e +ĠPro blem +ĠQu ad +Ġdecent ral +ĠV ector +an ut +Ġplug ins +ĠGreg ory +Ġfuck ed +el ines +ĠAmb assador +t ake +Ġcle ans +ong yang +An onymous +st ro +" } +al ine +ĠO dd +ĠE ug +2 16 +Ġbo il +ĠP owers +Ġnurs es +Ob viously +ĠTechn ical +Ġexceed ed +OR S +Ġextrem ists +Ġtr aces +ex pl +Ġcom r +ĠS ach +) / +Ġm asks +Ġsc i +B on +Ġreg ression +we gian +Ġadvis or +it ures +ĠV o +ex ample +ĠInst ruct +Ġs iege +Ġredu ctions +pt r +Ġstat utory +Ġrem oves +Ġp uck +red its +Ġbe e +Ġsal ad +Ġpromot ions +ĠJosh ua +with standing +ET H +ĠCh a +im us +Ġexpend iture +aun ting +Ġdelight ed +Ġ15 5 +be h +Ġcar pet +ĠSp art +Ġj ungle +l ists +Ġbull ying +ĠNob el +ĠGl en +Ġreferen ced +Ġintrodu ces +se in +Ġcho pped +gl ass +ĠW rest +Ġneutral ity +Ġâ Ļ +Ġinvestig ator +Ġshel ves +Ġun constitutional +Ġreprodu ction +Ġmer chant +m ia +Ġmet rics +Ġexplos ives +ĠSon ia +Ġbod ily +Ġthick ness +Ġpredomin antly +ĠAb ility +Ġmon itored +IC H +Ġ] . +ĠMart inez +Ġvis ibility +Ġqu eries +Ġgen ocide +ĠWar fare +Qu ery +Ġstud ios +Ġemb ry +Ġcorrid or +Ġclean ed +com plete +ĠM H +Ġenroll ment +ING S +Ġimpact ed +Ġdis astrous +ĠY un +ĠCl aire +ĠBas ically +y t +uster ity +Ġindirect ly +w ik +Ġd od +ĠCar r +Ġam p +Ġprohib it +ĠIn itial +ĠR d +ij i +Ġeduc ate +c orn +i ott +ĠBeaut y +Ġdetect ive +ĠCon n +s ince +Ġst agger +Ġob ese +Ġb ree +olog ic +is se +walk er +Ġbl ades +Ġlaw ful +fun c +ĠBeh ind +Ġappet ite +Ġ( * +Ġt ennis +Ġoff spring +Ġj ets +Ġstruct ured +Ġafore mentioned +N ov +Ġsc aling +f ill +Ġst ew +Ġcur b +ĠStep han +ed In +S F +ob ic +é ŃĶ +ou g +ĠM M +Ġgen etically +ope z +13 6 +Ġu mb +anc ers +Ġcoh ort +Ġmerch andise +Ġimp osing +ĠLegisl ature +ĠArch ive +iv ia +ĠN aval +Ġoff ences +Ġmir acle +Ġsn apped +Ġf oes +Ġextensive ly +ĠR af +Ġc ater +ed ience +K it +ĠB in +Ġrecomm ends +ĠC ities +Ġrig id +ĠRE AD +ĠNob le +ĠT ian +Ġcertific ates +ant is +o iler +ĠBudd hist +d id +Ġsurvey ed +Ġdown ward +Ġprint s +ĠMot ion +ron ics +ĠS ans +oss ibly +u ctions +Ġcolon ies +ĠDan ish +un it +Ġsp oil +Ġadvis ory +ber ries +Pl an +Ġspecific ation +op hers +ĠRes ource +Ġsh irts +prising ly +commun ications +Ġtriv ial +Ġmention ing +ise xual +Ġsupp lements +Ġsuper vision +B P +v or +Ġw it +Ġco oldown +Ġplaint iff +ĠReview s +ĠS ri +ĠM int +ĠSug ar +Ġafter ward +ĠPri est +ĠInvest ment +og ene +ĠT aking +Ġstretch ing +Ġinflamm ation +ĠTe hran +Ġl ining +Ġfree zing +ĠEnt ity +Ġins piring +spe cial +pr ice +Ġsu e +ĠP orter +oun ge +ET A +ĠD erek +ĠLu is +u o +ym ph +Ġex terior +ih il +ĠAsh ley +in ator +Ġnut rients +ĠTh rones +Ġfin ances +ĠIn spect +Ġspe cially +ĠRequ ired +ĠP TS +ĠViol ence +oint ed +sh ots +Ġex cerpt +co on +IN S +ĠG ri +Ġrecogn ised +We ek +You ng +Ġv om +is le +ĠCur ry +ĠBudd h +Ġnot ebook +Ġd urable +/ ? +ĠG ad +ĠP upp +Ġforg ive +p ark +Ġpersonal ities +an alysis +cl amation +Ġelev ator +Ġware house +ĠR ole +un n +Ġillust ration +ĠSc an +Ġatmosp heric +Im port +AN C +rict ed +f u +01 0 +Ġar che +Ġreward ed +akespe are +Ġintern ally +ĠR BI +alk er +Ġeleph ant +ow itz +ĠP izza +Ġbip artisan +é s +Ġslow ed +ĠSt ark +Ġover ride +OU S +Ġ3 20 +undred s +ĠDe ck +ĠC ensus +be e +14 6 +ot or +Ġ ip +Ġu b +oc ations +ĠBut ton +r ice +Ġc ripp +ff f +Ġorig inated +Ġoverwhel med +app a +Ġfore most +âĢ ij +ĠL EG +re lease +eat ured +at ches +Ġre ps +Ġl ending +ĠRe ference +ĠCl ient +16 5 +vent h +Com plete +ĠPat rol +Ġsw orn +c am +Ġshut tle +ĠR alph +Ġh ometown +- , +on al +ĠB P +å ı +Ġpersu ade +ĠAlex and +Ġcomb ines +Ġv ivid +ĠL ag +Ġenc oding +Ġsal vation +w en +ĠRec overy +i ya +Un iversity +ĠB iden +Ġbud gets +ĠTex ans +f its +Ġhon ored +Ġp ython +T D +## # +cl one +Ġbl ink +ĠL iquid +Ġunemploy ed +Ġcl ashes +ĠCoun sel +Ġdirect ing +Ġpun ct +ĠFal cons +Ġsh ark +ĠDam ascus +Ġje ans +Ġemb ark +Ġse ize +Ġup wards +2 80 +ĠE z +ĠAny thing +Ġex otic +l ower +ĠCreat or +ĠU m +Ġsubur bs +ber ger +ĠW end +Ġm int +ĠX X +ĠD ro +Ġsuff ers +Ġher b +t ree +Ġfrag ile +Ġflood ed +ĠAl cohol +ole an +ny der +ĠK O +F ram +Ġ13 6 +Ġow ed +ĠMe lee +ĠH ash +Ġwh isk +Ġsu do +r r +Qu ick +app ro +Ġi i +ĠEx amples +he e +Ġpromot es +per ature +k ar +ĠHon or +Ġs odium +ĠL if +ros so +intend ent +Ġcorrespond ent +F ound +sec ret +Ġident ifies +ag ne +Ġl ou +ĠP P +Ġcoinc idence +m ove +Ġmilit ia +Ġinf iltr +ĠPrim ary +Ġpitch ing +ĠI b +ĠGO OD +ãĤ ¸ +ĠW izards +ir al +ĠVen us +R R +ĠâĢ ķ +ĠCase y +Ġsad ly +Ġadm ire +Ġembarrass ed +c b +M el +Ġtub es +Ġbeaut ifully +ĠQueens land +Bel ow +re z +qu et +ple asant +Ġ « +C amp +Ġdec isive +19 98 +ĠL amb +ut ton +h n +ĠJ agu +au nder +ĠC ord +Ġcl erk +Ġca ffe +Ġwip ed +Ġre im +ĠMount ains +Ġimprison ed +Ġdevelop s +ĠP ra +Ġmodel ing +Any one +ance l +ĠS it +Ġshield s +Ġl awn +Ġcard iovascular +Ġdemonstr ating +Ġpar se +ĠIsrael is +Ġeuro s +14 3 +Ġgl orious +ins ki +ec d +Ġcondition ing +Ġhel pless +Ġmicro sc +ĠHar bor +Ġst akes +Ġ2 60 +Ġun equ +ĠFl oyd +Ġd amp +Ġappar atus +ĠLaw s +Ġcoun ters +Ġindu ce +at able +ĠAh med +Ġsl am +N ovember +Ġpers ist +Ġim minent +á n +Ġsh red +Ġph ases +ĠEd monton +ĠArm strong +ĠMe et +ĠK itty +Ñ Ģ +c irc +ĠAd ult +Ġa rose +ĠX en +D an +g ow +Ġsuper f +ĠAd mir +Ġend ure +Ġkey word +yr us +Ġy arn +Ġpath way +ĠHop kins +mid t +Ġcens orship +d ependent +Ġinstruct or +S ources +Ġto e +Ġball oon +N ob +Ġsw ear +ĠCast ro +Ġgl oss +ĠK avanaugh +Ġremark ably +Ph otos +ĠN om +ĠS outheast +y ers +Ġvalid ation +Ġcann on +ĠVict ory +ĠPier re +Ġcaut ious +Aud io +Ġf etch +ĠG ift +ĠH yp +Ġrem edy +Z E +Ġsc ent +Ġbe ard +ĠR ut +- " +Ġpat ents +H y +Ġun just +Ġpot ato +Ġforth coming +Ġche f +ĠR ift +aff e +ĠR OM +ĠL aunch +Ġp ads +ĠNe o +Ġon set +Ġsquee ze +s afe +Ġpref ix +ĠT M +ĠN early +ĠClin ical +ĠM ental +ot iation +ĠUn ic +ant ry +ĠC ir +Ġep it +à ¦ +Ġextract ed +verse ly +ri ad +Ġstr ains +Ġto ps +Ġpo em +ĠRand y +ĠMap le +TH ER +up iter +ĠSS D +ļ é +Ġun con +per ing +Ġsle pt +in ers +Ġunder water +ĠEv idence +g one +20 5 +Ġhistor ians +Ġsynt hesis +Ġf rog +b asketball +Ġvibr ant +Ġsub ord +Ġ3 65 +ĠD ial +Ġcooper ate +HA HA +Ġgreet ed +15 8 +Ġj azz +Ġinto x +ĠWalk ing +Ġsuper visor +ĠF usion +ĠMer cedes +s end +H am +s d +n l +Ġtour s +ĠF IFA +Ġcul p +g d +30 4 +Ġple as +Ġillust rates +ĠColomb ia +Ġhighlight ing +ĠSum mary +Ġexp osing +ĠD ru +Ġir ony +r itional +ĠCar roll +ĠEll is +P ict +ĠR apt +Ġad apter +Ġun m +Ġcor pse +Ġceleb rities +D en +at um +ĠAp ocalypse +ĠW ag +lin ing +Ġhorm ones +R ub +ĠX i +ĠV aults +20 8 +alky rie +inos aur +Ġfeed s +v ity +Ġdefe ating +W ait +Ġemphas ize +ĠSteel ers +yr inth +le ys +ĠWhe never +Current ly +ĠCl ock +Ġcollect ively +any on +ĠJ P +Ġment ality +Ġdownload s +Ġsurround ings +ĠBarn es +Ġflags hip +Ġindic ators +Ġgra pp +Jan uary +ĠElement al +ĠAthen a +ib al +Ġs ights +Ġcap ita +ĠTreat y +Ġvo iced +ĠG az +let te +Ġy a +Ġexp ired +Leg end +H ot +n ature +Ġunst able +Ġ2 80 +à º +Com ment +AL E +Ġquest s +Ġhand ler +n is +Ġvers atile +Ġconce al +enge ance +ĠInter active +Ġobs essed +ĠDog s +Ġcr acked +S ound +s v +ĠD ylan +ro ads +f x +ĠCath olics +ĠH ag +Ġsl ammed +Ġgl owing +s ale +Ġtiss ues +ĠCh i +ne e +Ġc her +s ic +ur rection +Ġb acon +ul atory +) ." +Ġir regular +FOR M +ass ed +Ġintention al +Ġcompens ate +ĠSpe aking +ĠS ets +15 3 +Ġconvent ions +b ands +em ade +Ġe cc +ĠWin ston +ĠAssass in +ĠBelg ian +Ġdepend ence +Ġnic he +Ġb ark +ĠJ azz +Ġdisadvant age +Ġgas oline +Ġ16 5 +çļ Ħ +ess a +mod ule +ang ular +O Y +ĠTreat ment +it as +ol ation +ĠArn old +Ġfe ud +ĠN est +Ġthe atre +ew ater +Ġmin ors +olic y +ĠH aven +div ision +Ġtr unk +F ar +ĠP ull +Ġcapt uring +Ġ18 00 +ĠTe en +Ġex empl +Ġclin ics +ĠB urg +Ġsubst it +Ġpay load +ĠL av +ĠT roy +ĠW itness +Ġfrag ments +Ġpass words +Ġg ospel +ĠG in +Ġten ants +ol ith +S ix +Pre vious +ĠAg es +ĠDar win +Ġbl at +Ġem pathy +sm ith +b ag +ĠE cho +ĠC amb +ĠM add +ĠB oo +Ġred e +ĠBurn ing +Ġsmooth ly +ĠAd rian +ĠV ampire +ĠMon sters +ste am +Sty le +M a +re a +ĠD war +aly st +urs or +Ġelim ination +Ġcrypt o +ch t +ĠE ternal +â̦ ] +ĠS orce +I ll +N ER +Ġu h +Con clusion +w age +Ġresp ir +Ġrem inis +het ical +Ġg y +Ġutil ized +ic idal +Ġ19 00 +Ġhun ters +ĠSw an +ĠRe act +Ġvis itor +ĠThanks giving +30 8 +Post s +Ġh ips +19 97 +om ers +Ġkn ocking +ĠVeh icle +Ġt il +Ġ13 8 +Ġm i +ĠInvest igation +ĠKen ya +Ġcas ino +Ġmot ives +Ġreg ain +re x +Ġweek ends +Ġstab bed +bor o +Ġexplo ited +ĠHA VE +ĠTe levision +c ock +Ġprepar ations +Ġende av +ĠRem ote +ĠM aker +ĠPro du +ĠEv an +Ġinform ational +ĠLouis ville +15 4 +ĠDream s +Ġpl ots +ĠRun ner +Ġhur ting +Ġacad emy +ĠMont gomery +n m +ĠL anc +ĠAl z +2 10 +el ong +Ġretail er +Ġar ising +Ġrebell ion +Ġbl onde +play ed +Ġinstrument al +C ross +Ġret ention +Ġtherape utic +Ġse as +Ġinfant ry +ĠCl int +Ġprompt ing +Ġbit ch +Ġst ems +ĠK ra +Ġthe sis +ĠB og +ru ed +Ġk ings +Ġcl ay +ific ent +ĠY ES +ĠTh ing +ĠCub s +vey ard +els h +in arily +ĠE y +ĠRoll ing +Ġev olving +Ind ia +Ġrecogn izes +Ġgrad uation +is ers +Ġfert ility +ĠMil an +Comm and +Ġbox ing +Ġ19 43 +Ġgl uten +ĠEm ir +Ġid ol +Ġcon ceived +ĠCre ation +Mer it +udd y +uss ions +ĠLie utenant +iet al +Ġunch anged +ĠSc ale +ĠCrime a +ball s +ator ial +Ġdepth s +Ġempir ical +Ġtrans m +Ġuns afe +miss ible +com fort +15 6 +Ġmechan ic +00 2 +l ins +Ġsm oked +P os +Ġslow ing +Ġl av +Tex as +Ġche ating +ĠMet ropolitan +eth yl +Ġdiscover ing +as se +Ġpen cil +ĠPy ongyang +Ġclos et +ĠShe et +ĠEnt ry +ou stic +Ġmy st +er ate +ari at +Ġminer als +Ġmusic ian +ĠP ul +ĠM az +24 9 +Ġper missions +Ġ iv +en ary +ick ers +ĠB ing +he a +en able +Ġgri ev +Ġassert ed +ĠColon el +Ġaff idav +w o +Ġse ated +ĠR ide +Ġpaint ings +ĠP ix +Ġ13 7 +ish i +umb ai +g otten +ĠEar l +Ġin ning +Ġc ensus +Ġtrave lled +ĠCons ult +18 5 +b ind +Ġsimpl icity +Ġoverlook ed +ĠHelp ful +Ġmon key +Ġoverwhelming ly +Bl ood +ĠFl int +ĠJ ama +ĠPres ent +ĠR age +ĠT A +pt ive +Ġturn out +w ald +ĠD olphins +ĠV PN +Ġon ion +Ġcraft ing +m ma +ĠMerc ury +Ġarr ange +Ġalert s +ĠO T +zb ollah +Ġg ases +ĠRichards on +s al +l ar +Ġfro st +Ġlower ing +Ġacc laim +Ġstart ups +ĠG ain +ess ment +Ġguard ian +äº º +ĠP ie +ĠL inks +Ġmer its +Ġaw ake +Ġparent al +Ġexceed s +Ġid le +ĠPil ot +Ġe Bay +ĠAc cept +ipe g +C am +ĠK ot +Ġtrad ers +olit ics +unk er +ĠP ale +os i +an mar +Ġ19 47 +ĠF ell +est ial +it ating +G F +ĠS r +if ted +Ġconnect or +ĠB one +ill es +2 60 +h ma +Ġoverl ap +ĠGit Hub +Ġclean er +ĠBapt ist +ĠW AS +Ġlung s +Ñ ģ +ĠB UT +Ġc ite +Ġpit ched +reat ment +Ġtro phies +ĠN u +38 6 +ĠPr ide +Ġattend ees +[ ] +17 9 +Ġspat ial +Ġpri zes +ĠRel igion +Ġshow case +ĠC ategory +vid ia +T arget +Pro perty +? , +Ġf usion +p ie +ĠU CLA +Ġsound track +Ġprin cess +ĠC aval +sh ould +Ġlim bs +Back ground +Ġlone ly +Ġc ores +ĠT ail +she et +Ġ13 2 +R a +ãĤ « +ĠB olt +Ġbook ed +Ġadmin ister +Ġequ als +w y +Ġobserv ing +ĠBar on +ĠAd obe +Ġv irgin +ĠSocial ist +M ove +gh azi +ĠLind a +2 12 +Ġbre wing +Ġmerch ants +bur se +Ġdiv or +Ġmet als +ĠN er +Ġsum s +ĠEn emy +Ġen vision +Ġgrant ing +ĠH oney +ĠSk yrim +Ġsoc io +gr aded +Ġselect ive +W ASHINGTON +Ġ19 48 +ĠSir ius +ĠG ross +act ivity +ĠI van +Ġfur ious +BS D +ĠPre vious +Ġrespons ive +Ġchar itable +Ġle aning +ĠP ew +Ġviol ates +\\\\ \\\\ +ĠCom ing +w ire +Ġpo et +Ġres olutions +comm and +ĠPortug uese +Ġnick name +Ġde af +Feb ruary +Ġrecogn ise +Ġentire ty +Ġseason al +pl aced +ĠTe legraph +Ġmicro phone +our ing +Ġgr ains +Ġgovern ed +Ġpost p +ĠW aters +in ement +Ġund ocumented +ĠCom cast +Ġf ox +Ġassault s +re on +man y +ĠJen kins +ĠAny way +Ġassess ments +Ġdown s +ĠM ouse +Ġsuper b +k t +ĠD ow +Ġtax ation +4 01 +Ġsm iles +Ġundert aken +Ġex h +Ġenthusi astic +Ġtw ent +Ġgovernment al +Ġautonom y +ĠTechn ologies +ĠCh ain +Ġpreval ent +f b +Ġnic otine +og ram +j ob +Ġawa iting +ĠMen u +Ġdep uties +k ov +ish ops +But ton +ĠShan ghai +Ġdies el +ĠD uck +R yan +ĠPC s +N F +j ury +ent e +Ġinacc urate +edd y +Wh atever +Ġshow c +ĠN ad +od us +et r +Ġplaint iffs +ĠW OR +ĠAss ange +Ġpriv at +Ġpremium s +Ġt am +UR L +Ġel ites +ĠR anger +otten ham +ĠH off +ĠAt hens +Ġdefin ite +Ġs ighed +Ġeven ly +2 11 +ĠAm ber +ak ia +Ġmail ing +Ġcr ashing +ĠConfeder ate +ru gged +W al +ĠDep ths +Ġjuven ile +Ġreact or +Introdu ction +ĠDel uxe +19 95 +ĠS anchez +ĠM ead +iv able +: - +ĠPlan ning +ĠT rap +qu in +ĠProt ect +ve red +In formation +Ġkid ney +inn amon +l as +Ġpolic ing +Ġtoler ate +ĠQ i +Ġbi ased +F ort +ĠK i +s ave +Ġprivile ged +Ġbe asts +ĠGl as +ĠC inem +Ġcome back +Sund ay +Ġext inction +h ops +Ġtrans mit +Ġdoub les +ĠFl at +16 7 +Ġdis puted +Ġinjust ice +f oo +V ict +role um +ĠJul ie +Con text +ĠR arity +iss ue +Comp onent +Ġcounsel ing +an ne +d ark +Ġobject ions +u ilt +Ġg ast +Ġpl ac +Ġun used +ãĥ ĩ +ĠT rial +ĠJ as +hed ral +ob b +Ġtempor al +ĠPR O +ĠN W +ĠAnn iversary +L arge +Ġther m +Ġd avid +Ġsystem ic +ĠSh ir +m ut +ĠNe pt +add ress +Ġscan ning +Ġunderstand able +Ġcan vas +C at +ĠZ oo +Ġang els +L O +ĠStat ement +ĠS ig +ov able +ĠA way +sh aring +ocr ats +st ated +Ġweigh ing +N or +w ild +B ey +Ġaston ishing +ĠReyn olds +Ġop ener +Ġtrain er +Ġsurg ical +p n +Ġadjust ing +whe el +Ġf rown +erv ative +Ġsusp end +With in +te in +Ġobst acle +Ġliber ties +ym es +Ġur anium +ans om +an ol +ub a +ĠL oss +Ġa rous +ĠHend erson +W ow +s pl +c ur +ĠÂ Ń +Ġtheir s +Dam age +Ġdownload ing +Ġdisc ern +ĠSt o +ĠFl a +Ġh ath +ĠA j +Ġun pleasant +Europe an +exp ensive +Ġscreens hot +ĠU V +Ġall ied +ĠPers ian +Ġmonop oly +Ġat om +ĠReds kins +"> < +Ġcan cell +Ġcinem a +13 1 +f air +ĠAlf red +Ġd uck +arg s +22 3 +ĠIS I +Ġsign aling +in ar +Ġlaugh s +Ġfor wards +Ġreck less +Ġlisten ers +at ivity +Ġvast ly +n ant +L ess +ĠHun ting +ĠScient ific +IT ED +Ġkn ight +ĠH TC +us a +t mp +Ġr ude +ĠLegend ary +Ġar ises +B ad +ĠCl aim +pe g +Ġreal ities +Th ink +Ġ ° +Ġro de +Ġstri ve +Ġan ecd +Ġshort s +Ġhypot hes +Ġcoord inated +ĠGand hi +ĠF PS +R ED +Ġsuscept ible +Ġshr ink +ĠCh art +Hel p +Ġ ion +de ep +rib es +ĠK ai +ĠCustom er +Sum mary +Ġc ough +w ife +Ġl end +Ġposition ing +Ġlot tery +ĠC anyon +Ġf ade +Ġbron ze +ĠKenn y +Ġbo asts +ĠEnh anced +rec ord +Ġemer gence +Ġa kin +ĠB ert +it ous +âĸ ij +Ġst ip +Ġexch anged +om ore +als h +Ġreserv oir +Ġstand point +W M +Ġiniti ate +Ġdec ay +Ġbrew ery +Ġter ribly +Ġmort al +lev ard +Ġrev is +N I +el o +Ġconf ess +ĠMS NBC +Ġsub missions +Cont roller +Ġ20 2 +ĠR uth +} ); +ĠAz ure +Ġ ." +20 6 +ĠMarket ing +Ġl aund +ien cies +Ġrenown ed +ĠT rou +ĠN GO +ble ms +Ġterr ified +Ġwar ns +Ġper t +Ġuns ure +4 80 +ale z +ult z +ĠOut side +Ġst yl +ĠUnder ground +Ġp anc +Ġd ictionary +Ġf oe +rim inal +ĠNor wegian +Ġj ailed +Ġm aternal +é e +ĠLu cy +c op +Ch o +Ġuns igned +ĠZe lda +ĠIns ider +ĠContin ued +Ġ13 3 +ĠNar uto +ĠMajor ity +16 9 +ĠW o +ãĤ ĵ +Ġpast or +Ġinform al +Ð ½ +an throp +jo in +ãģ Ĺ +it ational +N P +ĠWrit ing +f n +ĠB ever +19 5 +Ġy elling +Ġdr astically +Ġe ject +Ġne ut +Ġth rive +ĠFre qu +ou x +Ġpossess es +ĠSen ators +ĠD ES +ĠSh akespeare +ĠFran co +ĠL B +uch i +Ġinc arn +Ġfound ers +F unction +Ġbright ness +ĠB T +Ġwh ale +ĠThe ater +m ass +ĠD oll +S omething +Ġecho ed +ĠHe x +c rit +af ia +Ġgodd ess +Ġele ven +ĠPre view +ĠAur ora +Ġ4 01 +uls ive +ĠLog an +in burgh +ĠCent ers +ĠON LY +ĠA id +Ġparad ox +Ġh urd +ĠL C +D ue +c ourt +Ġoff ended +Ġeval uating +ĠMatthew s +Ġto mb +Ġpay roll +Ġextra ction +ĠH ands +if i +Ġsuper natural +ĠCOM M +] = +dog s +Ġ5 12 +ĠMe eting +Rich ard +ĠMax imum +Ġide als +Th ings +m and +ĠReg ardless +Ġhum ili +b uffer +L ittle +ĠD ani +ĠN ak +Ġliber ation +ĠA be +ĠO L +Ġstuff ed +ac a +ind a +raph ic +Ġmos qu +Ġcampaign ing +Ġoccup y +S qu +r ina +ĠW el +ĠV S +Ġphys ic +Ġp uls +r int +oad ed +ET F +ĠArch ives +Ġven ues +h ner +ĠTur bo +Ġl ust +Ġappeal ed +que z +il ib +ĠTim othy +Ġo mn +d ro +Ġobs ession +ĠSav age +19 96 +Gl obal +J es +2 14 +Ġsl iding +Ġdisapp ro +ĠMag ical +Ġvolunt arily +g b +ane y +Ġprop het +ĠRe in +ĠJul ia +ĠW orth +aur us +Ġb ounds +ie u +)) ) +Ġcro re +ĠCitiz en +S ky +Ġcolumn ist +Ġseek ers +ond o +IS A +ĠL ength +Ġnost alg +Ġnew com +Ġdet rim +ent ric +3 75 +ĠG E +Ġaut op +Ġacadem ics +App Data +ĠS hen +Ġid iot +ĠTrans it +Ġteasp oon +W il +K O +ĠCom edy +> , +Ġpop ulated +W D +Ġp igs +ĠO culus +Ġsymp athetic +Ġmar athon +19 8 +Ġseiz ure +s ided +Ġd op +irt ual +L and +ĠFl oor +osa urs +... ] +Ġl os +Ġsubsid iary +E Y +ĠPart s +ĠSt ef +ĠJud iciary +Ġ13 4 +Ġmir rors +Ġk et +t imes +Ġneuro log +Ġc av +ĠGu est +Ġtum or +sc ill +ĠLl oyd +E st +Ġcle arer +Ġstere otypes +Ġd ur +not hing +Red dit +Ġnegoti ated +---------------- -------- +23 5 +Ġfl own +ĠSe oul +ĠRes ident +ĠS CH +Ġdisappear ance +ĠV ince +g rown +Ġgrab s +r il +ĠInf inite +ĠTw enty +Ġpedest rian +Ġjer sey +ĠF ur +ĠInf inity +ĠEll iott +Ġment or +Ġmor ally +Ġob ey +sec ure +iff e +Ġantib iotics +ang led +ĠFre eman +ĠIntrodu ction +J un +Ġm arsh +ic ans +ĠEV ENTS +och ond +W all +icult y +Ġmisdem eanor +Ġl y +Th omas +ĠRes olution +Ġanim ations +ĠD ry +Ġinter course +ĠNew castle +ĠH og +ĠEqu ipment +17 7 +Ġterrit orial +Ġarch ives +20 3 +Fil ter +ĠMun ich +Ġcommand ed +ĠW and +Ġpit ches +ĠCro at +Ġrat ios +ĠM its +Ġaccum ulated +ĠSpecific ally +Ġgentle man +acer b +Ġp enn +Ġa ka +ĠF uk +Ġinterven e +ĠRef uge +ĠAlz heimer +Ġsuccess ion +oh an +d oes +L ord +Ġsepar at +Ġcorrespond ence +Ġsh iny +P rior +Ġs ulf +Ġmiser able +Ġded ication +( ). +Ġspecial ists +Ġdefect s +ĠC ult +ĠX ia +Ġje opard +ĠO re +Ab ility +Ġle ar +Ġamb itions +ĠB MI +ĠArab s +Ġ19 42 +Ġpres ervation +ific ate +Ġash amed +l oss +ĠRest aur +Ġrese mble +Ġen rich +ĠK N +ĠCl an +fl oat +Ġplay able +IT T +Ġharm ony +arr ison +ĠWe instein +w ere +Ġpoison ing +ĠCom put +ĠWord Press +m ajor +ĠVal ve +F an +ĠTh row +ĠRom ans +ĠDep ression +ad os +Ġtort ured +Ġbal ancing +bott om +Ġacqu iring +ĠMon te +ard i +Ġa ura +Ġ# # +ĠStand ing +ĠAtl as +C F +Ġintr ins +ĠBen ghazi +Ġcamp ing +Ġt apped +bl ade +st rous +ĠR abb +ĠW ritten +t ip +ĠNe igh +ster dam +ĠAll ow +ĠHe aling +ĠR hod +n um +Ġcaffe ine +ĠPer cent +Ġbo o +Ġapp les +30 5 +Ġwel coming +Ġappl aud +Ġa usterity + ± +ĠRe ality +ef e +å ® +Ġsu cks +Ġtab s +ĠPay Pal +Ġback pack +Ġgif ted +abul ary +ĠSc out +ir teen +Ġch in +Ġo mitted +Ġnegative ly +Ġaccess ing +ĠE arn +Ġambul ance +Ġhead phones +Ġ20 5 +ĠRef resh +p resident +ĠKit chen +ĠEnt ered +ĠS nyder +00 5 +om ical +Ġborrow ed +ĠN em +Ġav iation +Ġst all +rim ination +Ġuniform s +it ime +ĠSim mons +ener gy +ab lished +y y +qual ified +Ġrall ies +ĠSt uart +fl ight +Ġgang s +r ag +Ġv ault +lu x +ĠCom par +Ġdesign ation +20 9 +ĠJ os +d ollar +z ero +Ġwell s +30 3 +Ġconstitu ents +Ġhe ck +Ġc ows +Ġcommand ers +Ġdifferent ial +ĠC atherine +29 9 +Ġval ve +Ġbr ace +Ġperspect ives +c ert +f act +icular ly +ĠMc N +pl anes +Ġint ric +Ġpe as +ov an +Ġtoss ed +ret ch +ĠL opez +Ġunf amiliar +de ath +ĠA part +ĠCh ang +Ġrelie ved +rop he +Ġair ports +Ġfre ak +ut il +M ill +ĠCh in +ĠOw en +m ale +ĠBro ken +ĠWind s +ro b +r ising +Ġfire fighters +Ġauthor itarian +Ġ14 8 +Bit coin +ex ternal +Ġbrow sers +iche ver +or ian +Ġun b +Ġpo ke +ĠZ ot +M id +ĠPop ular +Ġco vert +Ġcont ributes +Ġ6 50 +Ġcont ention +G ate +Ġcons oles +Ġchrom os +ĠI X +Ġvis ually +ĠE isen +Ġjewel ry +Ġdeleg ation +Ġacceler ate +ĠR iley +Ġsl ope +Ġind oor +it ially +Ġhuge ly +Ġtun nels +Ġfin ed +Ġdirect ive +Ġfore head +ustom ed +Ġsk ate +Mus ic +g as +Ġrecogn izing +am bo +Ġover weight +ĠGr ade +Ù Ĭ +Ġsound ing +Ġlock ing +ĠR EM +St ore +Ġexc av +ĠLike wise +ĠL ights +Ġel bow +ĠSupp ly +w ic +Ġhands ome +19 94 +C oll +Ġadequ ately +ĠAssoci ate +Ġstri ps +Ġcrack down +Ġmar vel +ĠK un +Ġpass ages +@@ @@ +ĠT all +Ġthought ful +names e +Ġprost itution +bus iness +Ġball istic +person al +c ig +iz ational +R ound +ĠÂłĠÂł ĠÂłĠÂł +ĠCole man +Ġadm itting +ĠPl ug +Ġbit coins +ĠSu z +Ġfair ness +Ġsupp lier +Ġcatast rophic +ĠHel en +o qu +M arc +ĠArt icles +g ie +Ġend angered +Ġdest iny +ĠVol t +ol ia +ax is +Ġche at +Ġun ified +IC O +qu ote +30 2 +ĠS ed +Ġsupp ression +Ġanaly zing +Ġsqu at +Ġfig uring +Ġcoordin ates +Ġch unks +Ġ19 46 +Ġsub p +Ġw iki +ĠFor bes +ĠJ upiter +ĠE rik +im er +ĠCom mercial +\ ) +Ġlegitim acy +Ġd ental +ĠMe an +Ġdefic its +5 50 +Orig inally +ĠHor ror +Ġcontam ination +ll ah +Ġconf isc +ĠCl are +T B +ĠF ailed +an ed +Ġrul er +ĠCont roller +Ġfemin ists +F ix +g ay +20 7 +Ġr abbit +Th ird +ownt own +Ġgl ue +Ġvol atile +Ġsh ining +Ġf oll +Ġimp aired +Ġsup ers +æ Ī +Ġcl utch +ļé ĨĴ +Ġpro let +Ġ( ! +Ġy elled +ĠK iev +ĠEr n +ĠSh ock +K B +Ġsit uated +qu ery +ĠN as +Ġan nex +char acter +ĠHol iday +Ġautom ation +ĠJ ill +ĠRem astered +Ġl inem +Ġwild erness +ĠHor izon +ĠGu inea +A Z +Ġmain land +Ġsec recy +LE ASE +Ġp unk +ĠProv ince +( ), +Spe ed +Ġhand ing +ĠSeb ast +S ir +r ase +Ġj ournals +Ġcon gest +ĠT ut +ir rel +Ġschizophren ia +Ġmis ogyn +health y +I ron +Ġreact ed +- $ +25 2 +Ġpl ural +Ġpl um +Ġbarg ain +Ġground ed +f inder +Ġdis se +ĠL az +O OD +Ġat roc +F actory +Ġmin ions +Ġo ri +ĠB rave +ĠP RE +ĠMy anmar +ĠH od +Ġexped ition +Ġexpl ode +ĠCo ord +Ġext r +ĠB rief +ĠAD HD +Ġhard core +feed ing +Ġd ile +ĠF ruit +Ġvacc ination +ĠM ao +osp here +Ġcont ests +- | +Ġf ren +isp here +R om +ĠSh arp +ĠTre nd +Ġdis connect +âĢ¢ âĢ¢ +Ġper secution +Ear th +Ġhealth ier +38 4 +Ġc ob +ĠTr inity +OW S +AN N +Ġspecial ty +Ġg ru +Ġcooper ative +wh y +Start ing +ĠIss ues +st re +ens or +Ġ18 5 +Ad v +! ? +ĠRe vel +em ia +ĠH ulk +Ġcelebr ations +ĠS ou +ra ud +ĠKle in +Ġun real +con text +Ġpartners hips +Ġadop ting +t ical +Ġspl ash +ĠHe zbollah +c ategory +cycl op +xt on +ĠD ot +urd y +t z +Ġenvelop e +ĠN L +â ķ +Ġwhere in +Spe c +18 4 +Ġte lev +al iation +Ġmyth s +å ° +Ġrig orous +Ġcommun icating +Ġobser ver +Ġre he +ĠW ash +Ġapolog ized +ĠT in +Ġexpend itures +work ers +d ocument +Ġhes itate +ĠLen in +Ġunpredict able +Ġrenew al +cl er +ok ia +ĠCON T +Ġpost season +Tok ens +Ġex acerb +Ġbet ting +Ġ14 7 +Ġelev ation +W ood +ĠSol omon +19 4 +00 4 +out put +Ġredu nd +ĠM umbai +Ġp H +Ġreprodu ce +ĠD uration +MA X +Ġb og +C BS +ĠBal ance +ĠS gt +ĠRec ent +Ġc d +Ġpo pped +Ġincomp et +pro p +ay an +g uy +Pac ific +Ġty r +Ġ{ { +ĠMy stic +ĠD ana +Ġmast urb +Ġge ometry +à ¢ +ĠCor rect +Ġtraject ory +Ġdistract ed +Ġf oo +ĠW elsh +L uc +m ith +Ġrug by +Ġrespir atory +Ġtri angle +Ġ2 15 +Ġunder graduate +ĠSuper ior +ch anging +_ - +Ġright ly +Ġrefere e +Ġluc rative +Ġun authorized +Ġresemb les +ĠGN U +ĠDer by +Ġpath ways +ĠL ed +Ġend urance +Ġst int +Ġcollect or +F ast +Ġd ots +Ġnational s +ĠSec urities +Ġwh ip +Par am +Ġlearn s +M agic +Ġdetail ing +m oon +Ġbroadcast ing +Ġb aked +26 5 +hol m +ĠS ah +ĠHus sein +ĠCourt esy +17 4 +Ġ14 6 +Ġge ographic +pe ace +Ġjud ging +ĠS tern +B ur +Ġstory line +G un +ĠSt ick +24 5 +30 7 +ãĤ´ ãĥ³ +ĠAdminist rator +Ġbur nt +Ġp ave +ch oes +Ex ec +Ġcamp uses +Res ult +Ġmut ations +ĠCh arter +Ġcapt ures +Ġcomp ares +Ġbad ge +S cient +Ġer ad +ier y +o i +ett es +ĠE state +Ġst rap +Ġproud ly +Ġf ried +Ġwithd rawn +ĠV oy +ph ony +It ems +ĠP ierce +b ard +Ġann otation +ant on +ill on +Im pro +... ) +Ġhapp ier +---- -- +ad just +Ġstaff ers +Ġactiv ism +Ġper f +Ġal right +N eed +Ġcomm ence +Ġopio id +ĠAm anda +E s +ĠP ars +ĠK aw +W orks +24 8 +Ġind o +t c +end ant +ĠM oto +Ġlegal ization +OT E +Ġtask ed +Ġt sp +ĠACT IONS +16 6 +Ġrefres hing +ĠN R +ĠPere z +Ġinfring ement +S Y +List en +in ning +k u +Ġrot ate +pro gram +ar ah +Des ign +Ġ( £ +Ġst oring +Ġwar rants +Ġjud gement +ĠB rist +us ually +ph oto +ĠR an +ĠP ine +Ġoutrage ous +ĠValent ine +lu ence +ĠEvery body +Al tern +Ġrele vance +Ġtermin ated +Ġd essert +Ġfulf illed +Ġprosecut ed +ĠW ords +Ġm igrant +Ġcultiv ation +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +idel ity +ĠV ern +ĠLog in +Ġmetaph or +ĠT ip +Ġrecru its +ĠP ig +rib ing +Ġenthusi asts +ex per +Ġfright ening +ĠH air +ans on +str ate +Ġh i +He ight +Ġown ing +n one +Ġdis like +Ġkn ives +pher d +Ġloud ly +ĠAP Is +Dis play +ĠL ac +ĠUS S +ab l +ver ages +J ew +Ġ17 2 +ĠHist orical +at oon +ĠPhys ics +in tern +Ġwarm th +Ġto pp +D M +Ġgun man +Ġem peror +od i +ãĥ £ +in atory +ĠR ib +Ġ13 1 +ĠSat urn +ĠSh ining +Ġw aking +Qu otes +Ġcomed ian +en berg + ½ +Ġbelie vers +Ġpaper work +c ustom +Ġle v +Ġl ament +Ġpour ing +22 2 +p olitical +ĠSupp lement +m aid +Ġcruel ty +Ġt read +ys ics +A w +rit es +Ġmod ifier +ĠP osition +Ad am +l b +ub s +Ġimper fect +Ġcl usters +ĠEngine er +ĠC herry +Ġinaug uration +ĠS au +Ġembod iment +ĠUn cle +Ġover r +Ġexplos ions +c ule +ĠPrinc eton +ĠAndre a +Ġincorrect ly +Ġearn est +Ġpil gr +ĠS print +Ġslee ve +Ġhe ars +ĠAm azing +Ġbrow sing +ag in +Ġhom eland +Ġha w +Ġd iving +ist ered +17 8 +Ġbarg aining +ĠArc ade +Ġdeleg ate +ters on +................................ ................................ +ĠJackson ville +27 5 +Ġst agn +Ġad am +ĠSher man +C B +Ġsub urb +ĠFood s +Ġconver ting +ĠAr ist +Ġch ambers +l ove +Ġam ino +ĠG an +Ġmad ness +m c +ĠUS E +def ined +Ġul tr +ind ust +Ġw olves +l ance +Add itionally +Ġcr acks +as ia +ĠRe ason +ĠP ump +Ġaccident al +ĠL aser +ĠR id +Ġinitial ized +ell i +Ġun named +Ġn oun +ĠPass ed +Ġhost age +ĠEth iop +sh irts +Ġun rel +ĠEmb assy +Ġ19 41 +Ġat oms +Ġpur ported +16 4 +ĠF i +Ġgall ons +ĠMon ica +Ġp g +en ment +Ġsort ed +ĠG ospel +Ġhe ights +Ġtr aced +Ġunder going +She ll +Ġs acks +Ġproport ions +Ġhall uc +F ont +ac et +Ġwar mer +ĠIN TER +Ġgrab bing +Pl ug +Ġreal ization +ĠBur ke +Ġen chant +AT ER +ĠSe ed +Ġabund ant +F M +Ġc ivic +V s +is i +Ġv ow +Ġre per +ĠPartners hip +Ġpenet ration +Ġax e +Ġsh attered +ĠZ ombies +Ġv inyl +ĠAl ert +e on +Ġoblig ed +ĠIll ust +ĠPl aza +ĠFront ier +Ġdavid jl +ĠSer ial +ĠH av +ĠNut rition +B i +Ġâĸ Ī +ĠJ ays +lin ux +Ġhur ry +Ġv oy +Ġhop eless +ĠSte alth +Ġ ãģ +ess ors +tt le +b org +ĠSaf ari +f ell +Ġw ary +d ue +ĠAb ove +H a +E LL +Ġnot or +ĠW on +T oo +Ġoccup ations +Ġposs essions +Ġinv iting +Ġpred ators +Ġacceler ated +Ġ15 7 +uter te +ĠC ube +e ast +acc ount +G ive +Ġtrans plant +red ients +id able +Ġscreens hots +ĠG und +ĠF S +Ġtravel ers +Ġsens ory +ĠF iat +ĠRock ets +İ ĭ +_ { +F riend +Ġchar ming +AL S +Ġenjoy ment +m ph +Ġ5 000 +ĠRE G +Ù Ĩ +b ia +Ġcomp ilation +ro st +ĠV P +ĠSch ne +201 9 +Ġcop ying +M ORE +ĠFl ore +f alls +2 15 +t otal +Ġdis ciples +d ouble +Ġexceed ing +Ġsm ashed +Ġconcept ual +ĠRom ania +ĠB rent +ĠI CE +ĠT ou +Ġg rap +Ġn ails +18 9 +ãĥ ĺ +Ġproc ure +e ur +Ġconfir ming +ĠC ec +aw i +ĠEd en +Ġn g +Ġengine ered +at ics +Ġhook ed +Ġdisgust ing +ĠMur der +ãĤ ¿ +L ibrary +Ġ16 8 +Al most +hem atic +Men u +ĠNot re +ĠJ ur +Ġkidn apped +Ġhack er +ĠJ ade +Ġcreep y +Ġdraw ings +ĠSpons or +Ġcycl ists +ĠGob lin +Ġoptim ized +Ġst aged +ĠMc D +bet ween +A ge +en o +S ex +ĠW ide +n ings +av is +Ġincap able +ĠK ob +Ġreward ing +ĠL one +oles cent +Ġcontract ed +Ġstick y +J ose +B all +f est +ĠIn put +ĠRec ently +Ġto mat +squ are +App lication +Ġnit rogen +Ġdupl icate +ĠRec on +ĠD ear +L ondon +Ġint ra +Ġd ock +Ġout reach +ĠM illion +Ġmamm als +am pton +V AL +Ġsn aps +Ġd os +ĠWh ole +ĠRead y +T ry +ĠWinn ipeg +ear ance +Ġinc urred +ren ched +ĠNS W +il ot +rain e +Ġc ube +g ot +Ġrun way +etermin ed +ĠHaw ks +Ġsurviv or +ĠW ish +ĠD in +ĠDE F +ĠV ault +18 7 +Ġmush rooms +Ġcris p +be y +ĠDisco very +Ġdevelopment al +Ġparad igm +Ġcha otic +ĠT su +Ġ3 33 +b ons +Ġbacter ial +Ġcomm its +Ġcos mic +Ġme ga +oc ative +ĠP aint +ophob ic +Ġv ain +Ġcar ved +ĠTh ief +ĠG ul +ows hip +Ġc ites +ĠEd inburgh +Ġdimin ished +Ġacknowled ges +ĠK ills +Ġmic row +ĠHer a +Ġsen iors +Ġwhere by +H op +at ron +Ġun available +ĠN ate +Ġ4 80 +Ġsl ated +ĠRe becca +ĠB attery +Ġgram mar +Ġhead set +Ġcurs or +Ġex cluding +any e +aunder ing +eb in +Ġfeas ible +ĠPub lishing +ĠLab s +ĠCl iff +ĠFerr ari +Ġp ac +vis ible +mark ed +pe ll +Ġpol ite +Ġstagger ing +ĠGal actic +Ġsuper st +Ġpar an +ĠOffic ers +ãĢ ģ +Ġspecific s +ul us +23 9 +ĠP aste +AM P +ĠPan ama +ĠDe lete +angu ard +rest rial +Ġhero ic +ĠD y +ا ÙĦ +Ġincumb ent +Ġcr unch +t ro +Ġsc oop +Ġblog ger +Ġsell ers +ure n +Ġmedic ines +ĠC aps +ĠAnim ation +ox y +Ġout ward +Ġinqu iries +22 9 +Ġpsych ologist +ĠS ask +ev il +Ġcontam inated +ãĤ ¨ +he rence +Ġbrand ed +ĠAbd ul +z h +Ġparagraph s +Ġmin s +Ġcor related +er b +Ġimp art +Ġmil estone +ĠSol utions +ot le +Ġunder cover +Ġmar ched +ĠCharg ers +f ax +ĠSec rets +Ġr uth +we ather +Ġfemin ine +Ġsh am +Ġprest igious +igg ins +Ġs ung +hist ory +ett le +gg ie +Ġout dated +ol and +Ġper ceptions +ĠS ession +ĠDod gers +u j +ĠE ND +D oc +Ġdefic iency +Gr and +ĠJ oker +Ġretro spect +Ġdiagn ostic +Ġharm less +Ġro gue +ĠA val +E qu +Ġtrans c +ĠRoberts on +ĠDep ending +ĠBurn s +iv o +Ġhost ility +F eatures +ĵ ĺ +Ġdis comfort +ĠL CD +spec ified +ĠEx pect +3 40 +Ġimper ative +ĠReg ular +Ch inese +Ġstate wide +Ġsy mm +Ġlo ops +Ġaut umn +N ick +Ġsh aping +Ġqu ot +Ġc herry +ĠCross ref +è¦ ļéĨĴ +Stand ard +he ed +ĠD ell +ĠViet namese +Ġo st +ĠV alkyrie +O A +Ass ad +Ġreb ound +ĠTra ffic +pl aces +æ ĺ +ĠB uc +17 2 +Ġshel ters +Ġins isting +ĠCertain ly +ĠKenn eth +ĠT CP +Ġpen al +ĠRe play +he ard +Ġdial ect +iz a +ĠF Y +it cher +ĠD L +Ġspir al +Ġquarterback s +Ġh ull +Ġgo ogle +Ġto dd +ĠSter ling +ĠPl ate +Ġsp ying +mb ol +ĠReal m +ĠPro ced +ĠCr ash +Ġtermin ate +Ġprotest ing +C enter +gu ided +Ġun cover +Ġboy cott +Ġreal izes +s ound +Ġpret ending +ĠV as +19 80 +Ġfram ed +Ġ13 9 +Ġdesc ended +Ġrehab ilitation +Ġborrow ing +ĠB uch +Ġbl ur +R on +ĠFro zen +en za +Ch ief +ĠP oor +Ġtransl ates +M IN +Ġ2 12 +J ECT +Ġerupt ed +Ġsuccess es +S EC +Ġpl ague +Ġg ems +d oms +Ġstret ches +ĠSp y +Ġstory telling +C redit +ĠP ush +Ġtra ction +Ġin effective +ĠL una +Ġt apes +Ġanaly tics +erc ise +Ġprogram mes +ĠCar bon +Ġbeh old +he avy +ĠConserv ation +ĠF IR +Ġs ack +ter min +ric ks +Ġhous ed +Ġunus ually +I ce +Ġexecut ing +ĠMor oc +ed ay +Ġed itions +Ġsm arter +ĠB A +Ġout law +Ġvan ished +ib a +AL SE +ĠSil va +23 8 +C ould +Ġphilos opher +Ġevac uated +Sec ret +14 2 +Ġvis as +ãĤ ¬ +ĠM alt +ĠClear ly +ĠN iger +ĠC airo +ĠF ist +3 80 +ĠX ML +aut o +it ant +Ġrein forced +Rec ord +ĠSurviv or +G Hz +Ġscrew s +parent s +Ġo ceans +ma res +Ġbra kes +vas ive +Ġhell o +ĠS IM +rim p +Ġo re +ĠArm our +24 7 +Ġterr ific +Ġt ones +14 1 +ĠMin utes +Ep isode +Ġcur ves +Ġinflamm atory +Ġbat ting +ĠBeaut iful +L ay +Ġunp op +v able +Ġr iots +ĠTact ics +b augh +ĠC ock +Ġorg asm +ĠS as +Ġconstruct or +et z +G ov +Ġant agon +Ġthe at +Ġde eds +ha o +c uts +ĠMc Cl +Ġu m +ĠScient ists +Ġgrass roots +ys sey +"] => +Ġsurf aced +Ġsh ades +Ġneighb ours +Ġad vertis +oy a +Ġmer ged +Up on +Ġg ad +Ġanticip ate +Any way +Ġsl ogan +Ġdis respect +I ran +ĠT B +act ed +Ġsubp oen +medi ately +OO OO +Ġwa iver +Ġvulner abilities +ott esville +ĠHuff ington +J osh +ĠD H +M onday +ĠEll en +K now +x on +it ems +22 8 +Ġf ills +ĠN ike +Ġcum ulative +and als +I r +Ġ ì +Ġfr iction +ig ator +Ġsc ans +ĠVi enna +ld om +Ġperform ers +P rim +Ġb idding +M ur +Ġlean ed +ĠPri x +al ks +Ġ[ â̦] +ĠTw itch +ĠDevelop er +ĠG ir +Ġcall back +Ab stract +Ġacc ustomed +Ġfreed oms +ĠP G +ur acy +Ġl ump +is man +,, ,, +19 92 +ĠR ED +Ġwor m +M atch +ĠPl atinum +I J +ĠOwn er +Tri via +com pl +Ġnew born +Ġfant as +O wn +Ġ19 59 +Ġsymp ath +Ġub iqu +Ġoutput s +Ġal lev +Ġpr ag +K evin +Ġfav ors +Ġbur ial +Ġn urt +so lete +c ache +Ġ15 6 +Ġunl ocks +te chn +M aking +Ġcon quer +ad ic +æ ĸ +Ġel f +Ġelect orate +ĠKurd s +ĠSt ack +ĠSam urai +Ġâ ĺħ +Ġ{ } +ĠS aid +ĠFall out +Ġkind ness +ĠCustom s +ĠBou levard +Ġhelicop ters +ot ics +ĠVe get +com ment +Ġcritic ised +Ġpol ished +ĠRem ix +ĠC ultural +Ġrec ons +Ġdo i +at em +Sc reen +Ġbar red +Com ments +ĠGener ally +Ġsl ap +7 20 +V ari +p ine +Ġem pt +Ġh ats +ĠPlay ing +l ab +a verage +form s +ĠC otton +Ġcan s +ĠD ON +ĠSom alia +C rypt +ĠIncre ases +E ver +mod ern +Ġsur geon +3 000 +Ġrandom ized +================================ ================================ +B ern +im pl +ĠC OR +Ġpro claim +th ouse +Ġto es +Ġam ple +Ġpres erving +Ġdis bel +gr and +B esides +Ġsil k +ĠPat tern +h m +Ġenter prises +Ġaffidav it +ĠAdvis ory +Ġadvert ised +ĠRel igious +se ctions +psy ch +ĠField s +aw ays +Ġhasht ag +ĠNight mare +Ġv ampire +Ġfore nsic +rosso ver +n ar +Ġn avy +Ġvac ant +ĠD uel +Ġhall way +Ġface book +ident ally +ĠN RA +Ġm att +Ġhur ricane +ĠKir by +ĠP uzzle +Ġsk irt +ou st +du llah +Ġanal ogy +in ion +Ġtomat oes +ĠN V +ĠPe ak +ĠMe yer +Ġappoint ments +Ġm asc +Ġal ley +re hend +Ġchar ities +Ġund o +Ġdest inations +ĠTest ing +"> " +c ats +* . +Ġgest ures +gener al +Le ague +Ġpack ets +ĠInspect or +ĠBer g +Ġfraud ulent +Ġcritic ize +F un +Ġbl aming +nd ra +Ġsl ash +ĠE ston +Ġpropos ing +Ġwh ales +Ġtherap ist +Ġsub set +Ġle isure +EL D +ĠC VE +ĠAct ivity +Ġcul min +sh op +ĠD AY +is cher +ĠAdmir al +ĠAtt acks +Ġ19 58 +Ġmem oir +Ġfold ed +Ġsex ist +Ġ15 3 +ĠL I +Ġread ings +Ġembarrass ment +ĠEmploy ment +w art +ch in +Ġcontin uation +l ia +Rec ently +Ġd uel +Ġevac uation +ĠKash mir +Ġdis position +ĠR ig +Ġbol ts +Ġins urers +4 67 +M ex +Ġret aliation +Ġmis ery +Ġunre asonable +r aining +I mm +ĠP U +em er +Ġgen ital +ãĤ ³ +ĠC andy +Ġon ions +ĠP att +lin er +Ġconced ed +Ġf a +Ġfor c +ĠH ernandez +ĠGe off +deb ian +ĠTe ams +Ġc ries +Ġhome owners +23 7 +A BC +Ġst itch +Ġstat istic +Ġhead ers +ĠBi ology +Ġmot ors +ĠG EN +ĠL ip +Ġh ates +Ġhe el +S elf +i pl +ED IT +ort ing +Ġann ot +ĠSpe ech +old emort +ĠJ avascript +ĠLe Bron +Ġfoot print +Ġf n +Ġseiz ures +n as +h ide +Ġ19 54 +ĠBe e +ĠDecl aration +ĠKat ie +Ġreserv ations +N R +f emale +Ġsatur ated +Ġb iblical +Ġtroll s +Dev ice +ph otos +Ġdr ums +ãĥīãĥ© ãĤ´ãĥ³ +N ight +f ighter +ĠH ak +ri ber +Ġc ush +Ġdiscipl inary +ba um +ĠG H +ĠSch midt +ilib rium +Ġs ixty +ĠKush ner +ro ts +Ġp und +ĠR ac +Ġspr ings +Ġcon ve +Bus iness +F all +Ġqual ifications +Ġvers es +Ġnarc iss +ĠK oh +ĠW ow +ĠCharl ottesville +ed o +Ġinterrog ation +ĠW ool +36 5 +B rian +Ġâľ ĵ +Ġalleg es +ond s +id ation +ĠJack ie +y u +Ġl akes +Ġworth while +Ġcryst als +ĠJud a +Ġcomp rehend +Ġfl ush +Ġabsor ption +ĠO C +Ġfright ened +ĠCh ocolate +Mart in +Ġbu ys +Ġbu cks +Ġapp ell +ĠChampions hips +Ġlist ener +ĠDef ensive +Ġc z +ud s +ĠM ate +Ġre play +Ġdecor ated +Ġs unk +ĠV IP +ĠAn k +Ġ19 5 +aa aa +Nob ody +ĠMil k +ĠG ur +ĠM k +ĠS ara +Ġse ating +ĠW id +Tr ack +Ġemploy s +Ġgig antic +AP P +ãĤ § +in ventory +Ġtow el +at che +l asting +ĠT L +Ġlat ency +Ġkn e +B er +me aning +Ġup held +Ġplay ground +Ġm ant +S ide +Ġstere o +Ġnorth west +Ġexception ally +Ġr ays +Ġrec urring +D rive +Ġup right +Ġab duct +ĠMar athon +Ġgood bye +Ġal phabet +h p +Ġcourt room +ring ton +ot hing +T ag +Ġdiplom ats +Ġbar bar +ĠAqu a +18 3 +33 33 +Ġmat urity +Ġinst ability +ĠAp ache +Ġ= == +Ġfast ing +ĠGr id +Mod Loader +Ġ15 2 +A bs +ĠOper ating +ett i +Ġacqu aint +Don nell +ĠK em +ĠFor ge +Ġarm ored +M il +Ġphilos ophers +in vest +Pl ayers +â Ī +Ġmy riad +Ġcomr ades +R ot +Ġremember ing +Ġcorrespond s +Ġprogram mers +ĠLyn n +Ġo lig +Ġco herent +yn chron +ĠChem ical +Ġj ugg +p air +post s +E ye +ĠIn ner +Ġsem ester +ott est +ĠEmir ates +ric anes +or ously +m its +ĠW is +Ġd odge +l ocation +Ġf aded +Am azon +ĠPro ceed +ĠIN FO +j ournal +ĠTru ck +T en +Ġ2 17 +Ġstat utes +m obile +ĠT ypes +Rec omm +b uster +pe x +Ġleg ends +Ġhead ache +f aced +ĠWi Fi +if ty +ĠH ER +Ġcirc uits +ER ROR +22 6 +ol in +Ġcyl inder +osp ace +ik ers +P rem +Qu ant +Ġconflic ting +Ġslight est +Ġfor ged +ion age +Step hen +ĠK ub +ĠOpp ortun +ĠHe al +Ġbl o +Ġrul ers +Ġh uh +Ġsubmar ine +f y +ass er +Ġallow ance +ĠKas ich +ĠT as +ĠAustral ians +Forge ModLoader +ĠâĨ ij +ĠMat rix +am ins +Ġ12 00 +ĠAc qu +23 6 +D ocument +ĠBre aking +19 3 +ĠSub st +ĠRoll er +ĠPro perties +ĠN I +t ier +Ġcr ushing +Ġadvoc ating +Further more +keep ers +Ġsex ism +x d +Ġcall er +ĠS ense +chie ve +ĠT F +Ġfuel ed +Ġreminis cent +Ġobs ess +ur st +Ġup hold +ĠF ans +het ics +Ġâ Ĺ +ĠB ath +Ġbe verage +Ġo scill +25 4 +Ġpol es +Ġgrad ual +Ġex ting +ĠS uff +ĠS uddenly +Ġlik ing +Ġ19 49 +un ciation +am ination +ĠO mar +ĠL V +ĠCon sequently +Ġsynt hes +ĠG IF +Ġp ains +Ġinteract ing +u ously +inc re +Ġrum or +ĠScient ology +19 7 +ĠZ ig +Ġspe lling +ĠA SS +Ġexting u +ms on +Ġg h +Ġremark ed +ĠStrateg ic +ĠM ON +å ¥ +g ae +ĠWH AT +E ric +ĠCamp us +Ġmeth ane +Ġimag in +J UST +ĠAl m +X T +i q +ĠR SS +Ġwrong doing +att a +Ġbig ot +Ġdemonstr ators +ĠCal vin +ĠV illa +Ġmembr ane +ĠAw esome +Ġbenef ic +26 8 +Ġmagn ificent +ĠL ots +G reg +ĠBor is +Ġdetain ees +ĠH erman +Ġwhis pered +Ġa we +Prof essor +fund ing +Ġphys iological +ĠDest ruction +Ġlim b +Ġmanip ulated +Ġbub bles +Ġpse ud +Ġhyd ra +ĠBrist ol +Ġst ellar +ĠExp ansion +ĠK ell +ĠInterest ingly +Ġm ans +Ġdrag ging +Ġec ological +ĠF it +Ġg ent +Ġbenef ited +ĠHait i +Ġpoly g +ãĥ İ +Ġ20 30 +Ġpro w +Ġrecon struction +Ġwas t +Ġpsych ic +ĠGree ks +Hand ler +16 2 +ĠP ulse +Ġsol icit +Ġsy s +Ġinflu x +ĠG entle +per cent +Ġprolifer ation +Ġtax able +Ġdisreg ard +Ġesc aping +Ġg inger +Ġwith stand +Ġdevast ated +ĠD ew +ser ies +Ġinject ed +ela ide +Ġturn over +he at +Ļ Ĥ +H appy +ĠSil ent +ãĤ Ń +iv ism +Ġir rational +AM A +Ġre ef +r ub +Ġ16 2 +Ġbank ers +ĠEth ics +v v +Ġcritic isms +K n +18 6 +M ovie +ĠT ories +Ġno od +Ġdist ortion +F alse +od ore +Ġt asty +Res earch +ĠU ID +- ) +Ġdivor ced +ĠM U +ĠHay es +ĠIs n +ian i +ĠH Q +Ġ" # +ign ant +Ġtra umatic +ĠL ing +H un +Ġsab ot +on line +r andom +Ġren amed +ra red +K A +d ead +é t +ĠAss istance +Ġse af +++++ ++++ +Ġse ldom +ĠWeb b +Ġbo olean +u let +Ġref rain +ĠDI Y +ru le +Ġshut ting +Ġutil izing +load ing +ĠPar am +co al +oot er +Ġattract ing +ĠD ol +Ġher s +ag netic +ĠRe ach +im o +Ġdisc arded +ĠP ip +01 5 +ü r +Ġm ug +Im agine +C OL +Ġcurs ed +ĠSh ows +ĠCurt is +ĠSach s +spe aking +ĠV ista +ĠFram ework +ong o +Ġsub reddit +Ġcr us +ĠO val +R ow +g rowing +Ġinstall ment +Ġgl ac +ĠAdv ance +EC K +ĠLGBT Q +LE Y +Ġac et +Ġsuccess ive +ĠNic ole +Ġ19 57 +Qu ote +Ġcircumst ance +ack ets +Ġ14 2 +ort ium +Ġguess ed +ĠFr ame +Ġperpet rators +ĠAv iation +ĠBen ch +Ġhand c +A p +Ġ19 56 +25 9 +r and +Net Message +d in +urt les +h ig +ĠV III +ff iti +ĠSw ords +b ial +Ġkidn apping +dev ice +Ġb arn +ĠEl i +auc as +S end +Con structed +Ġ ½ +Ġneed les +Ġad vertisements +Ġv ou +Ġexhib ited +ĠFort ress +As k +B erry +TY PE +Ġcan cers +ump ing +ĠTerrit ory +Ġpr ud +Ġn as +Ġathe ist +Ġbal ances +ãģ Ł +ĠSh awn +& & +Ġland sc +ĠR GB +Ġpet ty +Ġex cellence +Ġtransl ations +Ġpar cel +ĠChe v +E ast +ĠOut put +im i +Ġamb ient +ĠTh reat +Ġvill ains +Ġ5 50 +IC A +Ġtall er +Ġle aking +c up +Ġpol ish +Ġinfect ious +ĠK C +Ġ@ @ +back ground +Ġbureaucr acy +ĠS ai +un less +it ious +ĠSky pe +At l +ID ENT +00 8 +Ġhyp ocr +Ġpit chers +Ġguess ing +ĠF INAL +Bet ween +Ġvill agers +Ġ25 2 +f ashion +ĠTun is +Be h +ĠEx c +ĠM ID +28 8 +ĠHas kell +19 6 +ĠN OR +Ġspec s +Ġinv ari +Ġgl ut +ĠC ars +Ġimp ulse +Ġhon ors +g el +Ġjurisd ictions +ĠBund le +ul as +Calif ornia +ĠIncre ase +Ġp ear +Ġsing les +Ġc ues +Ġunder went +ĠW S +Ġexagger ated +Ġdub ious +Ġfl ashing +L OG +) ]. +J ournal +t g +V an +ĠI stanbul +ĠIn sp +ĠFrank en +D raw +Ġsad ness +Ġiron ic +ĠF ry +x c +Ġ16 4 +is ch +W ay +ĠProtest ant +h orn +Ġun aff +ĠV iv +ill as +ĠProduct ions +ĠH ogan +Ġper imeter +ĠS isters +Ġspont aneous +Ġdown side +Ġdescend ants +Ġor n +w orm +Japan ese +Ġ19 55 +Ġ15 1 +ĠDo ing +els en +umb les +Ġrad ically +ĠDr um +ĠB ach +Ġli abilities +ĠO B +ĠElement ary +Ġmem e +yn es +Ġfinger print +ĠGr ab +Ġundert ake +Mem bers +ĠRead er +ĠSim s +g od +Ġhypot hetical +s cient +ĠA J +Ġchar ism +Ġad missions +ĠMiss ile +tr ade +Ġexerc ising +ĠBack ground +W ritten +Ġvoc als +whe ther +Ġv i +ĠW inner +Ġl itter +ĠSh ooting +ST EM +ãĤ ¡ +ĠA FL +Ġvari ability +Ġe ats +ĠD PS +b row +Ġeleph ants +Ġstr at +Ġ Å +Ġsett lers +Matt hew +Ġin advert +H I +ĠIM F +ĠGo al +Ġnerv es +John son +ey e +ablish ment +Th ursday +BIL ITY +H ad +am oto +het amine +ep s +Ġmit ochond +Ġcomp ressed +ĠTre vor +ĠAnim als +T ool +L ock +Ġtwe ak +Ġpin ch +Ġcancell ation +P ot +Ġfoc al +ĠAst ron +17 3 +ĠA SC +ĠO THER +umn i +Ġdem ise +d l +Ù ħ +Sem itism +Ġcr acking +Ġcollabor ative +Ġexpl ores +s ql +Ġher bs +Ġconfig urations +m is +ĠRes ult +ace y +ĠSm oke +Ġsan ct +el ia +Ġdeg ener +Ġdeep est +Ġscream ed +Ġn ap +Soft ware +ĠST AR +E F +ĠX in +spons ored +mans hip +23 3 +Ġprim aries +Ġfilter ing +Ġas semble +m il +ĠMy ers +b ows +Ġpun ched +M ic +Ġinnov ations +Ġfun c +and o +Ġfr acking +ĠV ul +о Ð +osh op +ĠIm mun +Ġsett ling +Ġadolesc ents +Ġreb uilding +Ġtransform ing +Ġpar ole +Ġhar bor +Ġbook ing +ot ional +onge vity +ĠY o +b ug +Ġemer ges +ĠMethod s +ĠCh u +P res +ĠDun geons +Ġtra iling +ĠR um +ĠH ugh +å¤ © +ĠE ra +ĠBatt les +Res ults +ĠTr ading +Ġvers a +c ss +ax ies +he et +Ġgre ed +19 89 +Ġgard ens +Ġconting ent +P ark +ĠLeaf s +h ook +ro be +Ġdiplom acy +ĠF uel +ĠInv asion +Ġupgr ading +M ale +Ġe lic +Ġrelent less +ĠCo venant +ap esh +ĠT rop +T y +pro duction +art y +Ġpun ches +ak o +cyclop edia +ĠR abbit +ĠHD MI +Ġ14 1 +Ġf oil +Item Image +ĠF G +Ġimplement ations +ĠP om +ixt ures +Ġaw ait +Ġ3 30 +am us +Ġumb rella +Ġfore see +se par +Ġcircum cision +Ġperipher al +S ay +ĠExper t +In c +Ġwithd rew +ĠAnd ers +f ried +Ġradio active +ĠOp ening +Ġboard ing +ĠN D +Ġover throw +Act iv +W P +ĠAct s +× Ļ +Ġmot ions +v ic +ĠM ighty +ĠDef ender +a er +Ġthank ful +ĠK illing +ĠBr is +mo il +Ġpredict ing +26 6 +ch oice +Ġkill ers +Ġinc ub +ĠChe st +ather ing +Ġpro claimed +fl ower +oss om +umbled ore +ĠCy cling +ĠOccup y +AG ES +P en +ĠY ug +Ġpack aged +Ġheight ened +c ot +st ack +C ond +Ġst amps +m age +Ġpersu aded +Ġens l +ĠCard inal +Ġsol itary +Ġpossess ing +ĠC ork +Ġev id +ĠT ay +Ġbl ues +Ġextrem ism +Ġlun ar +Ġcl own +Te chn +Ġfest ivals +ĠPv P +ĠL ar +Ġconsequ ently +p resent +Ġsom eday +ç İĭ +ĠMet eor +Ġtour ing +c ulture +Ġbe aches +S hip +c ause +ĠFl ood +ãĥ ¯ +Ġpur ity +th ose +Ġem ission +b olt +Ġch ord +ĠScript ure +L u +Ġ$ { +cre ated +Other s +25 8 +Ġelement al +Ġannoy ed +ĠA E +d an +ĠS ag +Res earchers +Ġfair y +âĢĵ âĢĵ +======== ==== +Sm art +GG GG +Ġskelet ons +Ġpup ils +link ed +Ġur gency +en abled +ĠF uck +Ġcoun cill +r ab +U AL +T I +Ġlif es +Ġconf essed +B ug +Ġharm on +ĠCON FIG +ĠNe utral +D ouble +Ġst aple +ĠSH A +Brit ish +ĠSN P +AT OR +oc o +Ġswing ing +ge x +ole on +pl ain +ĠMiss ing +ĠTro phy +v ari +ran ch +Ġ3 01 +4 40 +00000000 00000000 +Ġrest oring +Ġha ul +uc ing +ner g +Ġfut ures +Ġstrateg ist +quest ion +Ġlater al +ĠB ard +Ġs or +ĠRhod es +ĠD owntown +????? - +ĠL it +ĠB ened +Ġco il +st reet +ĠPort al +FI LE +ĠG ru +* , +23 1 +ne um +Ġsuck ed +Ġr apper +Ġtend encies +ĠLaure n +cell aneous +26 7 +Ġbrow se +Ġover c +head er +o ise +Ġbe et +ĠG le +St ay +Ġm um +Ġtyp ed +Ġdiscount s +T alk +ĠO g +ex isting +ĠS ell +u ph +C I +ĠAust rian +ĠW arm +Ġdismiss al +Ġaver ages +c amera +Ġalleg iance +L AN +=" # +Ġcomment ators +ĠSet ting +ĠMid west +Ġpharm ac +ĠEX P +Ġstain less +Ch icago +Ġt an +24 4 +Ġcountry side +ĠV ac +29 5 +Ġpin ned +Ġcr ises +Ġstandard ized +T ask +ĠJ ail +ĠD ocker +col ored +f orth +" }, +Ġpat rons +Ġsp ice +Ġm ourn +ĠM ood +Ġlaund ry +Ġequ ip +ĠM ole +y ll +ĠTH C +n ation +ĠSher lock +Ġiss u +ĠK re +ĠAmeric as +ĠA AA +Ġsystem atically +Ġcont ra +ĠS ally +Ġrational e +Ġcar riage +Ġpe aks +Ġcontrad iction +ens ation +ĠFail ure +Ġpro ps +Ġnames pace +Ġc ove +field s +ãĤ ĭ +Ġw ool +ĠC atch +Ġpresum ed +ĠD iana +r agon +ig i +Ġh amm +Ġst unt +ĠG UI +ĠObserv atory +ĠSh ore +Ġsmell s +ann ah +Ġcock pit +ĠD uterte +8 50 +Ġopp ressed +bre aker +ĠCont ribut +ĠPer u +ĠMons anto +ĠAtt empt +Ġcommand ing +Ġfr idge +ĠR in +ĠChe ss +ual ity +Ġo l +Republic an +ĠGl ory +ĠW IN +.... ... +ag ent +read ing +Ġin h +J ones +Ġcl icks +al an +Ġ[ ]; +ĠMaj esty +ĠC ed +op us +ate l +à ª +AR C +ĠEc uador +ãĥ ł +ĠK uro +Ġritual s +Ġcapt ive +Ġoun ce +Ġdisag reement +Ġsl og +f uel +P et +M ail +Ġexerc ised +Ġsol ic +Ġrain fall +Ġdev otion +ĠAss essment +Ġrob otic +opt ions +ĠR P +ĠFam ilies +ĠFl ames +Ġassign ments +00 7 +aked own +Ġvoc abulary +Re illy +Ġc aval +g ars +Ġsupp ressed +ĠS ET +ĠJohn s +Ġwar p +bro ken +Ġstat ues +Ġadvoc ated +Ġ2 75 +Ġper il +om orph +ĠF emin +per fect +Ġh atch +L ib +5 12 +Ġlif elong +3 13 +Ġche eks +Ġnum bered +ĠM ug +B ody +ra vel +We ight +ĠJ ak +ĠHe ath +Ġkiss ing +ĠJ UST +Ġw aving +u pload +Ġins ider +ĠPro gressive +ĠFil ter +tt a +ĠBe am +Ġviol ently +ip ation +Ġskept icism +Ġ19 18 +ĠAnn ie +ĠS I +Ġgen etics +Ġon board +at l +ĠFried man +ĠB ri +cept ive +Ġpir ate +ĠRep orter +27 8 +Ġmyth ology +Ġe clipse +Ġsk ins +Ġgly ph +ing ham +F iles +C our +w omen +Ġreg imes +Ġphotograp hed +K at +ĠMA X +Offic ials +Ġunexpected ly +Ġimpress ions +F ront +;;;; ;;;; +Ġsuprem acy +Ġs ang +Ġaggrav ated +Ġabrupt ly +ĠS ector +Ġexc uses +Ġcost ing +ide press +St ack +ĠR NA +ob il +Ġghost s +ld on +at ibility +Top ics +Ġreim burse +ĠH M +ĠDe g +Ġth ief +y et +ogen esis +le aning +ĠK ol +ĠB asketball +Ġf i +ĠSee ing +Ġrecy cling +Ġ[ - +Cong ress +Ġlect ures +P sy +Ġne p +Ġm aid +Ġori ented +A X +Ġrespect ful +re ne +fl ush +ĠUn loaded +re quest +gr id +ĠAltern atively +ĠHug o +Ġdec ree +ĠBuddh ism +and um +And roid +ĠCong o +ĠJoy ce +Ġacknowled ging +hes ive +ĠTom orrow +ĠH iro +th ren +ĠM aced +Ġho ax +ĠIncre ased +ĠPr adesh +W ild +____ __ +16 1 +Ġa unt +Ġdistribut ing +ĠT ucker +ĠSS L +ĠW olves +B uilding +ou lt +ĠLu o +ĠY as +ĠSp ir +ĠSh ape +ĠCamb od +ĠIP v +Ġm l +Ġext rad +39 0 +ĠPenn y +d ream +Ġstation ed +opt ional +ew orthy +. +ĠWorks hop +ĠRet ail +ĠAv atar +6 25 +N a +ĠV C +ĠSec ure +M Y +19 88 +oss ip +Ġpro state +Ġund en +Ġg amer +ĠCont ents +ĠWar hammer +ĠSent inel +3 10 +Ġse gregation +ĠF lex +ĠM AY +Ġdr ills +ĠDrug s +Islam ic +Ġsp ur +Ġca fe +Ġimag inary +Ġgu iding +Ġsw ings +ĠThe me +ob y +Ġn ud +Ġbe gging +Ġstr ongh +Ġreject ing +Ġpedest rians +ĠPro spect +R are +s le +Ġconcess ions +ĠConst itutional +Ġbe ams +Ġfib ers +p oon +Ġinstinct s +pro perty +ĠB IG +Sand ers +im ates +Ġco ating +Ġcorps es +ĠTR UE +check ed +Ġ16 6 +A sh +ĠJ S +ĠF iction +Ġcommun al +Ġener getic +oooo oooo +Ġnow adays +IL D +ib o +ĠSU V +R en +Ġdwell ing +Sil ver +Ġt ally +ĠM oving +Ġcow ard +Ġgener als +Ġhorn s +Ġcirc ulated +Ġrob bed +ĠUn limited +Ġharass ed +Ġinhib it +Ġcomp oser +ĠSpot ify +Ġspread s +3 64 +Ġsu icidal +Ġno ises +ĠSt ur +Ġs aga +ĠK ag +is o +Ġtheoret ically +M oney +Ġsimilar ity +Ġslic ed +ut ils +ing es +" - +Ġan th +Ġimp ed +Mod ule +Through out +Ġmen us +comm ittee +and i +ob j +in av +f ired +ĠAb dullah +Ġund ead +Ġfont s +H old +EN G +Ġsustain ability +Ġfl ick +Ġr azor +ĠF est +ĠChar acters +Ġword ing +Ġpopul ist +Ġcritic izing +Ġm use +v ine +Ġcard board +Ġkind ly +Ġfr inge +ĠThe ft +icult ural +Ġgovern ors +Ġ ���� +Ġ16 3 +Ġtime out +ĠA uth +Child ren +A U +Ġred emption +ĠAl ger +Ġ19 14 +Ġw aved +Ġastron auts +og rams +Ġsw amp +ĠFinn ish +Ġcand le +Ġton nes +ut m +Ġr ay +Ġsp un +Ġfear ful +art icles +Ġca us +or ically +ĠRequ ires +ĠG ol +Ġpop e +Ġinaug ural +Ġg le +AD A +ĠIS IL +ĠOff ensive +Ġwatch dog +Ġbal con +ent ity +ĠH oo +Ġgall on +AC C +Ġdoub ling +Ġimpl ication +ĠS ight +Ġdoct r +---- --- +Ġ\ \ +Ġm alt +R oll +Ġâī ¥ +Ġrec ap +add ing +u ces +ĠB end +fig ure +Ġtur key +Ġsoc ietal +ĠT ickets +Ġcommer cially +Ġsp icy +Ġ2 16 +ĠR amp +Ġsuperior ity +à ¯ +ĠTr acker +C arl +ĠC oy +ĠPatri ot +Ġconsult ed +Ġlist ings +Ġsle w +reens hot +ĠG one +Ġ[ ...] +30 9 +Ġh ottest +Ø ± +Ġrock y +ĠD iaz +Ġmass age +Ġpar aly +Ġp ony +A z +Ġcart ridge +ĠN Z +Ġsn ack +ĠLam ar +ple ment +ĠLes lie +Ġm ater +Ġsn ipp +24 6 +Ġjoint ly +ĠBris bane +ĠiP od +Ġpump ing +Ġgo at +ĠSh aron +eal ing +Ġcor on +Ġan omal +rah im +ĠConnect ion +Ġsculpt ure +Ġsched uling +ĠD addy +at hing +Ġeyeb rows +Ġcur ved +Ġsent iments +Ġdraft ing +D rop +( [ +Ġnom inal +ĠLeaders hip +ĠG row +Ġ17 6 +Ġconstruct ive +iv ation +Ġcorrupt ed +ger ald +ĠC ros +ĠChe ster +ĠL ap +ãģ ª +OT H +D ATA +Ġal mond +pro bably +I mp +Ġfe ast +ĠWar craft +F lor +Ġcheck point +Ġtrans cription +Ġ20 4 +Ġtwe aks +Ġrel ieve +S cience +Ġperform er +Z one +Ġtur moil +ig ated +hib it +ĠC afe +the med +Ġflu or +ben ch +Ġde com +ĠU nt +ĠBar rett +ĠF acts +Ġt asting +ĠPTS D +ĠSe al +ĠJuda ism +ĠDynam ic +ĠC ors +V e +ĠM ing +ĠTrans form +v on +ĠDef enders +ĠTact ical +ĠV on +ĠUn ivers +Ġdist orted +ĠB reath +?' " +Ġag on +ĠDead ly +Ġl an +ĠCy cle +orn ed +Ġrel iably +Ġgl or +ĠMon key +ãĥ ¡ +Ġad ren +Ġmicrow ave +ĠAl ban +irc raft +dig it +sm art +ĠD read +¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ +{ { +ĠRoc hester +Ġsimpl ified +Ġinf licted +Ġtake over +Ġyour selves +ad itional +Ġmus cular +K S +Ġing en +T ax +ĠFe ature +27 7 +Ġcru c +Ġcr ate +Ġun identified +Ġacclaim ed +ĠM anga +ĠFr ances +ĠNep al +ĠG erald +ĠKu wait +Ġsl ain +ĠHe b +ĠG oku +ãģ® æ +28 6 +M rs +ĠC ody +ĠSan ctuary +01 6 +Ġdism ant +Ġdatas et +ĠH ond +b uck +ĠPat terson +Ġpal ette +ĠG D +ic ol +ĠL odge +Ġplanet ary +ak in +ĠRegist ered +ab we +ĠPeters burg +Ġha iled +ĠP iece +S che +ĠDO J +Ġen umer +18 1 +ĠObs erver +ĠB old +f ounded +com merce +Ġexplo its +ĠF inding +UR N +ĠS ne +ĠAc id +ay ette +ĠVal ues +Ġdr astic +Ġarchitect ural +Ġ" . +× ķ +ump ed +Ġwra pping +Ġwid ow +ĠSl ayer +l ace +on ce +German y +av oid +Ġtem ples +P AR +à ´ +ĠLuc ifer +ĠFl ickr +l ov +for ces +Ġsc outing +Ġlou der +tes y +Ġbefore hand +Ä ĵ +ĠNe on +ĠW ol +ĠTyp ically +ĠPolit ico +-+ -+ +Ġbuild er +Ġder ive +K ill +Ġp oker +Ġambig uous +Ġlif ts +Ġcy t +Ġrib s +ood le +ĠS ounds +h air +ĠSynd rome +t f +Ġproport ional +u id +Ġper taining +ĠKind le +ĠNeg ro +Ġreiter ated +ĠTon ight +oth s +ĠCorn ell +Ġo wing +Ġ20 8 +elf are +oc ating +ĠB irds +Sub scribe +Ġess ays +Ġburd ens +Ġillust rations +ar ious +ER AL +ĠCal cul +Ġx en +ĠLink edIn +ĠJ ung +Ġredes ign +Con nor +29 6 +Ġrevers al +ĠAd elaide +ĠL L +Ġs inking +Ġg um +US H +c apt +ĠGr imm +Ġfoot steps +ĠCB D +isp ers +Ġpro se +Wed nesday +ĠM ovies +ed in +Ġoverturn ed +Ġcontent ious +US B +~~~~~~~~ ~~~~~~~~ +ĠCo pper +Ġpoint less +N V +val ues +olph in +d ain +Ġdepos ited +ĠG W +Ġpreced ed +ĠCl a +ĠGo lem +ĠN im +ĠÎ ² +ĠEngine ers +m iddle +Ġfl att +oper ative +Ġcouncil s +imb abwe +el in +Ġstress ful +ĠL D +Ġres h +l ake +Ġwheel chair +ĠAltern ative +Ġoptim ize +oper ation +Ġpe ek +Ġones elf +ig il +Ġtrans itions +op athy +bl ank +Ġ16 9 +17 1 +________________________________ ________________________________ +Ġl aundering +En c +ĠD EC +Ġwork outs +Ġsp ikes +Ġdin osaurs +Ġdiscrim inatory +P ool +R ather +38 5 +R NA +tes ters +et o +ĠIdent ity +Ġve in +ĠBur ton +Ġarc ade +4 20 +Ult imately +ĠSad ly +à ° +p ill +Ġcub ic +ĠSpect rum +the se +st ates +Ġun official +h awks +ĠEVER Y +Ġrain bow +Ġincarcer ation +and ing +Ġsy ll +ĠEver ton +Ġ17 9 +ĠSer bia +Ġ18 9 +m eter +ĠMic key +Ġant iqu +Ġfact ual +ne ck +ĠN are +n orm +m ust +Ġhigh ways +Ġgl am +Ġdivid ing +ĠSquad ron +ĠMar tha +Ġbirth s +C over +//////// //////// +ĠW ong +Ph ot +ĠA LS +ri o +ĠNon etheless +ĠL emon +Ġ20 6 +ĠE E +Ġderiv ative +ĠWW II +v ote +Ġthere in +Ġsepar ating +44 6 +sy nc +ĠStre ets +Ġr att +Ġmunicip ality +ĠShort ly +Ġmon k +) ," +Ġscr ub +Ġoper atives +Ne ither +Pl ace +ĠLim it +F emale +ĠAct or +Char acter +Ġconstit uted +35 7 +Ġprotest ed +ĠSt raw +ĠHe ight +ild a +ĠTy ph +Ġflood s +Ġcos metic +W AY +pert ure +up on +t ons +ess ing +ĠP ocket +Ġro oft +ĠC aucas +Ġant idepress +Ġincomp atible +EC D +Ġoper a +ĠCont est +Ġgener ators +l ime +Def ense +19 87 +for um +Ġsav age +ĠHung arian +n z +Ġmet allic +Ġex pelled +Ġres idency +Ġdress es +66 6 +ĠC lement +f ires +C ategory +Ġge ek +al is +Ġc emetery +educ ated +Ġc rawl +ĠUn able +ĠT yson +ak is +Ġp ardon +ĠW ra +Ġstrengthen ed +ĠF ors +33 5 +ĠH C +ĠM ond +Ġvisual s +ĠBeat les +ett lement +Ġ ï +g ro +Ġb ash +Ġpo orest +Ġex cel +Ġaspir ations +ĠM unicip +ens ible +Ġceremon ies +Ġintimid ation +ĠCON TR +be ck +ĠK ap +as u +Ġtradem arks +ĠS ew +ĠComp etition +net work +ĠAr ri +ĠT et +Ro aming +W C +D at +Ġso b +Ġpair ing +Ġoverd ose +SA Y +ab er +Ġrev olt +ĠF ah +act ing +e q +est ation +F ight +ĠMar ks +27 3 +Ġ17 8 +R aw +ãģ ĭ +34 9 +bl ocks +Ġver ge +est ine +ĠPod esta +Ġinv asive +Ġprofound ly +ĠA o +e ach +Ġl est +inter pret +Ġshr inking +Ġerr one +Ġche es +ly s +ĠI vy +ĠDirect ory +Ġhint ed +V ICE +Ġcontact ing +ĠG ent +he i +Ġlabel ing +Ġmerc ury +ĠL ite +Ġexp ires +Ġdest abil +rit is +c u +Ġfeather s +Ġste er +Ġprogram med +ĠV ader +Go ing +ĠE lim +Ġy o +ĠMic he +Ġ20 3 +Ġslee ves +Ġb ully +ĠHum ans +36 8 +Ġcomp ress +ĠBan ner +AR S +Ġa while +Ġcal ib +Ġspons orship +ĠDiff iculty +ĠP apers +Ġident ifier +} . +Ġy og +ĠSh ia +Ġclean up +Ġvib e +int rodu +im ming +Austral ia +Ġout lines +ĠY outube +tr ain +ĠM akes +Ġde ported +Ġcent r +ĠD ug +ĠB oulder +ĠBuff y +Ġinj unction +ĠHar ley +ĠG roups +ĠD umbledore +ĠCl ara +Ġ" - +Ġsacrific ed +ep h +Sh adow +ib ling +Ġfreel ance +Ġevident ly +ph al +Ġret ains +M ir +Ġfin ite +d ar +ĠC ous +Ġrep aired +Ġperiod ic +Ġchampions hips +Ġaster oid +bl ind +Ġexpress ly +ĠAst ros +Ġsc aled +Ġge ographical +ĠRap ids +En joy +Ġel astic +ĠMoh amed +Mark et +be gin +Ġdisco vers +Ġtele communications +Ġscan ner +Ġen large +Ġsh arks +Ġpsy chedel +ĠRou ge +Ġsnap shot +is ine +X P +Ġpestic ides +ĠL SD +ĠDist ribution +re ally +Ġde gradation +Ġdisgu ise +Ġbi om +ĠEX T +Ġequ ations +Ġhaz ards +ĠComp ared +) * +Ġvirt ues +Ġeld ers +Ġenh ancing +ĠAc ross +er os +ang ling +Ġcomb ust +ucc i +Ġconc ussion +Ġcontrace ption +ĠK ang +Ġexpress es +Ġa ux +ĠP ione +Ġexhib its +Deb ug +OT AL +ĠAl ready +ĠWheel er +Ġexp ands +? : +Ġreconc iliation +Ġpir ates +Ġpur se +Ġdiscour age +Ġspect acle +R ank +Ġwra ps +ĠTh ought +Ġimp ending +O pp +ĠAng lo +ĠE UR +Ġscrew ed +ret ched +Ġencour agement +mod els +Ġconf use +mm m +ĠVit amin +âĸij âĸij +C ru +Ġkn ights +Ġdisc ard +Ġb ishops +ĠW ear +ĠGar rett +k an +ãĥ Ł +Ġmascul ine +cap ital +ĠA us +Ġfat ally +th anks +ĠA U +ĠG ut +12 00 +Ġ 00000000 +Ġsur rog +ĠBI OS +ra its +ĠWat ts +Ġresur rection +ĠElect oral +ĠT ips +4 000 +Ġnut rient +Ġdepict ing +Ġspr ink +Ġm uff +ĠL IM +ĠS ample +ps c +ib i +gener ated +Ġspec imens +Ġdiss atisf +Ġtail ored +Ġhold ings +ĠMonth ly +ĠE at +po ons +Ġne c +ĠC age +ĠLot us +ĠLan tern +Ġfront ier +Ġp ensions +Ġj oked +ĠHard y +=-=- =-=- +r ade +U ID +Ġr ails +Ġem it +Ġsl ate +Ġsm ug +Ġsp it +ĠCall s +ĠJac obs +f eat +ĠU E +Ġrest ruct +Ġregener ation +Ġenerg ies +ĠCon nor +OH N +ĠChe ese +Ġg er +Ġresur rect +man agement +N W +Ġpres ently +ĠBru ins +M ember +ĠM ang +id an +Ġboost ing +w yn ++ . +requ isite +ĠNY PD +ĠMe gan +ĠCond itions +Ġp ics +nes ium +ĠR ash +Ġ17 4 +ĠD ucks +Ġemb ro +z u +on ian +rel igious +Ġc raz +ĠAC A +ĠZ ucker +EM A +ĠPro s +We apon +ĠKn ox +ĠAr duino +Ġst ove +Ġheaven s +ĠP urchase +Ġher d +Ġfundra iser +Dig ital +5 000 +Ġprop onents +/ âĢĭ +Ġj elly +ĠVis a +Ġmon ks +Ġadvance ment +ĠW er +Ġ18 7 +e us +ert ility +Ġfet al +Ġ19 36 +L o +Ġout fits +Ġstair case +b omb +Ġcustom ized +cl air +T ree +Ġm apped +ĠConsider ing +ĠTor res +Ġmeth yl +Ġapprox imate +Ġdo om +ĠHans en +Ġc rossover +Ġstand alone +ä ¼ +Ġinv ites +Ġgra veyard +Ġh p +Donald Trump +Ġesc ort +G ar +Ġpredec essors +Ġh ay +Ġen zyme +ĠStra ight +vis ors +I ng +ane ously +ĠApp lied +Ġf ec +ĠDur ant +Ġout spoken +or b +Ġz eal +Ġdisgr ace +' ). +ĠChe ng +28 9 +ĠRen a +ĠSu icide +29 4 +Ġout raged +ĠNew man +ĠN vidia +ĠA ber +ĠB ers +Ġrecre ation +Wind ow +ĠD P +x e +Ġped oph +Ġfall out +ambo o +Ġpresent ations +ĠApp s +Ġh tml +3 45 +ĠX XX +Ġrub bing +ĠLe ather +Ġhum idity +se ys +est ablished +ĠUn its +64 6 +Ġrespect able +A uto +Ġthri ving +ĠInn ovation +ang s +Ext ra +reg ulation +29 8 +p ick +Ex amples +ĠC J +Att ack +Ġdr acon +L T +Ġstick er +re rs +Ġsun ny +I ss +reg ulated +d im +ĠAb stract +Ġhus bands +Off ice +om ination +it ars +AN GE +asc al +ĠK ris +ĠInf antry +Ġm alf +ĠA the +ĠR ally +bal anced +................ ........ +OU P +Ġmole cule +met ics +ĠSpl it +ĠInstruct ions +ĠN ights +c ards +Ġt ug +Ġcon e +å Ń +Ġt x +ĠDisc ussion +Ġcatast rophe +pp e +g io +Ġcommun ism +Ġhal ted +ĠGu ant +cle an +ĠSc hed +ĠK anye +Ġw ander +ĠSer iously +Ġ18 8 +enn ial +f ollow +product ive +ĠFl ow +ĠS ail +Ġc raw +Ġsim ulations +or u +ang les +ĠN olan +Ġmen stru +4 70 +Ġ20 7 +aj a +Ġcas ually +board ing +Ġ2 22 +ov y +ĠN umbers +um at +O E +28 7 +ĠCle mson +Ġcert s +Ġsl id +ĠT ribe +Ġto ast +Ġfort unes +Ġf als +ĠComm ittees +Ġg p +Ġf iery +ĠN ets +ĠAn ime +Pack age +ĠComp are +l aughter +in fect +Ġatroc ities +Ġjust ices +Ġins ults +ĠVern on +Ġsh aken +Ġperson a +est amp +36 7 +br ain +Ġexperiment ing +K en +ĠElect ronics +Ġ16 1 +dom ain +Ġgraph ical +b ishop +Ġwho pping +ĠEv angel +Ġadvertis ers +ĠSpe ar +Ġb ids +Ġdestro ys +ut z +Ġunders c +ĠAD D +Ġan ts +ĠC um +ipp les +ĠF ill +Ġgl anced +Ġind icted +ĠE ff +Ġmis con +ĠDes ktop +Ġab ide +ãĥ Ģ +ĠI o +ĠC oul +Ġcaps ule +ĠCh rys +M ON +Ġund es +ĠI RA +Ġc itation +Ġdict ate +ĠNet works +ĠConf lict +ĠSt uff +x a +is ec +ĠChem istry +Ġquarter ly +William s +an an +O pt +ĠAlexand ria +out heastern +ĠSpring field +ĠBlack s +Ġge ography +24 2 +Ġut most +ĠEx xon +ab outs +E VA +ĠEn able +ĠBar r +Ġdisag reed +ĠCy prus +Ġdement ia +Ġlab s +Ġubiqu itous +ĠLO VE +Ġconsolid ated +s r +Ġcream y +ĠTim ber +Reg ardless +ĠCert ificate +Ġ" ... +ogen ous +Capt ain +Ġinsult ing +ĠSor os +ĠInst r +ĠBulgar ia +bet ter +Ġsuck ing +ĠDavid son +at z +Ġcoll ateral +g if +Ġplag ued +ĠC ancel +ĠGard ner +R B +Ġsix teen +Rem ove +ur istic +c ook +R od +Ġcompr ising +f le +) âĢĶ +ĠVik ing +g rowth +agon al +Ġsr f +af ety +m ot +N early +st own +ĠF actor +Ġautom obile +Ġproced ural +m ask +amp ires +Ġdisapp ears +j ab +3 15 +Ġ19 51 +ne eded +Ġd aring +le ader +Ġp odium +Ġun healthy +Ġm und +Ġpy ramid +oc re +Ġkiss ed +Ġdream ed +ĠFant astic +ĠG ly +å Ĭ +Ġgreat ness +Ġsp ices +Ġmet ropolitan +Ġcomp uls +i ets +101 6 +ĠSh am +ĠP yr +fl ies +ĠMid night +Ġswall owed +Ġgen res +ĠL ucky +ĠRew ards +Ġdisp atch +ĠI PA +ĠApp ly +Ġa ven +al ities +3 12 +th ings +Ġ( ). +Ġm ates +ĠS z +ĠC OP +ol ate +O FF +Ġre charge +c aps +ĠYork er +ic one +Ġgal axies +ile aks +D ave +ĠP uzz +ĠCelt ic +ĠA FC +27 6 +ĠS ons +Ġaffirm ative +H or +Ġtutorial s +ĠC ITY +ĠR osa +ĠExt ension +Ser ies +Ġf ats +Ġr ab +l is +Ġun ic +Ġe ve +ĠSp in +Ġadul thood +ty p +Ġsect arian +Ġcheck out +ĠCy cl +S ingle +Ġmart yr +Ġch illing +88 8 +ou fl +Ġ] ; +Ġcongest ion +m k +ĠWhere as +Ġ19 38 +ur rencies +er ion +Ġbo ast +ĠPat ients +Ġch ap +ĠB D +real DonaldTrump +Ġexam ines +h ov +Ġstart ling +ĠBab ylon +w id +om ew +br ance +ĠOd yssey +w ig +Ġtor ch +ĠV ox +ĠMo z +ĠT roll +ĠAn s +Similar ly +ĠF ul +00 6 +Un less +ĠAl one +st ead +ĠPub lisher +r ights +t u +ĠDoes n +Ġprofession ally +Ġcl o +ic z +Ġste als +Ġ á +19 86 +Ġst urdy +ĠJoh ann +Ġmed als +Ġfil ings +ĠFr aser +d one +Ġmult inational +Ġf eder +Ġworth less +Ġp est +Yes terday +ank ind +Ġg ays +Ġb orne +ĠP OS +Pict ure +Ġpercent ages +25 1 +r ame +Ġpot ions +AM D +ĠLeban ese +Ġr ang +ĠL SU +ong s +Ġpen insula +ĠCl ause +AL K +oh a +ĠMac Book +Ġunanim ous +Ġl enders +Ġhang s +Ġfranch ises +ore rs +ĠUp dates +Ġisol ate +and ro +S oon +Ġdisrupt ive +ĠSur ve +Ġst itches +ĠSc orp +ĠDomin ion +Ġsupp lying +Ar g +Ġtur ret +ĠL uk +Ġbr ackets +* ) +ĠRevolution ary +ĠHon est +Ġnot icing +ĠSh annon +Ġafford ed +Ġth a +ĠJan et +! -- +ĠNare ndra +ĠPl ot +H ol +se ver +e enth +Ġobst ruction +Ġ10 24 +st aff +j as +or get +sc enes +l aughs +ĠF argo +cr ime +Ġorche str +Ġde let +ili ary +rie ved +Ġmilit ar +ĠGreen e +âĹ ı +ãģ ¦ +ĠGu ards +Ġunle ashed +ĠWe ber +Ġadjust able +Ġcal iber +Ġmotiv ations +Ġà ł +m Ah +ĠL anka +hand le +Ġp ent +ĠR av +ĠAng ular +ĠK au +umb ing +Ġphil anthrop +Ġde hyd +Ġtox icity +e er +ĠY ORK +w itz +å ¼ +ĠI E +commun ity +ĠA H +Ġret ali +Ġmass ively +ĠDani els +ĠD EL +Ġcar cin +Ur l +Ġrout ing +ĠNPC s +ĠR AF +ry ce +Ġwa ived +ĠGu atem +Every body +Ġco venant +Ġ17 3 +Ġrelax ing +Ġqu art +al most +Ġguard ed +ĠSold iers +ĠPL AY +Ġout going +L AND +Ġre write +ĠM OV +ĠIm per +ĠS olution +Ġphenomen al +Ġl ongevity +Ġimp at +ĠN issan +ir ie +Ġod or +ĠZ ar +ok s +Ġmilit ias +ĠSP EC +Ġtoler ated +ars er +ĠBrad ford ++ , +Ġsur real +s f +Can adian +Ġresemb lance +Ġcarbohyd rate +VI EW +Ġaccess ory +me al +larg est +ieg el +Some one +Ġtoug hest +os o +Ġfun nel +Ġcondemn ation +lu ent +Ġw ired +ĠSun set +Jes us +ĠP ST +ĠP ages +ĠTy coon +ĠP F +Ġselect ions +Ġ ठ+part isan +Ġhigh s +ĠR une +Ġcraft s +le ad +ĠParent s +Ġre claim +ek er +ĠAll ied +ae per +Ġlo oming +Ġbenefic iaries +ĠH ull +Stud ents +Jew ish +d j +Ġp act +tem plate +ĠOffic ials +ĠBay lor +Ġhe mp +Ġyouth s +ĠLevel s +ĠX iao +ĠC hes +Ġende avor +ĠRem oved +Ġhipp ocamp +H ell +ãĤ Ĭ +80 5 +Ġd inosaur +ĠWr ath +ĠIndones ian +Ġcalcul ator +ĠD ictionary +Ġ4 20 +ĠM AG +( _ +! , +t arians +Ġrestrict ing +rac use +Ġweek day +OU NT +Ġsh rugged +leg round +Ġb ald +ĠDo ctors +Ġt outed +ĠMax well +Ġ2 14 +Ġdiplom at +Ġrep ression +Ġconstitu ency +v ice +r anked +ĠNap oleon +g ang +ĠFore ver +t un +Ġbul b +ĠPD T +ĠC isco +V EN +Ġres umed +Ste ven +ĠManit oba +Ġfab ulous +ĠAg ents +19 84 +Ġam using +ĠMyster ies +Ġor thodox +fl oor +Ġquestion naire +Ġpenet rate +Ġfilm makers +ĠUn c +Ġst amped +Ġth irteen +Ġout field +Ġforward ed +Ġapp ra +Ġa ided +t ry +Ġunf ocused +ĠL iz +ĠWend y +ĠSc ene +Ch arg +Ġreject s +Ġleft ist +ĠProv idence +ĠBr id +reg n +Ġprophe cy +ĠL IVE +4 99 +Ġfor ge +ĠF ML +Ġintrins ic +ĠF rog +Ġw ont +ĠH olt +Ġfam ed +CL US +aeper nick +ĠH ate +ĠC ay +Ġregister ing +ort ality +rop y +ocaly ptic +a an +n av +Ġfasc ist +IF IED +Ġimpl icated +ĠRes ort +ĠChand ler +ĠBr ick +P in +ys c +Us age +ĠHel m +us ra +âĺħ âĺħ +ĠAb bas +Ġunanim ously +Ġke eper +Ġadd icted +?? ? +Ġhelm ets +Ġant ioxid +aps ed +80 8 +gi ene +Ġwa its +Ġmin ion +ra ved +ĠP orsche +Ġdream ing +Ġ17 1 +ĠC ain +Ġun for +ass o +ĠConfig uration +k un +hard t +Ġn ested +ĠL DS +L ES +Ġt ying +en os +Ġc ue +ĠMar qu +sk irts +Ġclick ed +Ġexp iration +ĠAccording ly +ĠW C +Ġbless ings +Ġaddict ive +ĠN arr +y x +ĠJagu ars +Ġrent s +ĠS iber +Ġt ipped +ous se +ĠFitz gerald +Ġhier arch +out ine +Ġwa velength +> . +ch id +ĠProcess ing +/ + +r anking +E asy +ĠConst ruct +Ġt et +ins ured +H UD +Ġqu oting +Ġcommun icated +in x +Ġin mate +Ġerect ed +ĠAbs olutely +ĠSure ly +Ġun im +ĠThr one +he id +Ġcl aws +Ġsuper star +ĠL enn +ĠWh is +U k +ab ol +Ġsk et +ĠN iet +Ġper ks +Ġaff inity +Ġopen ings +phas is +Ġdiscrim inate +T ip +v c +Ġgr inding +ĠJenn y +Ġast hma +hol es +ĠHom er +Ġreg isters +ĠGl ad +Ġcre ations +Ġlith ium +Ġappl ause +unt il +Just ice +ĠTur ks +Ġsc andals +Ġb ake +t ank +M ech +ĠMe ans +ĠM aid +Republic ans +is al +wind ows +ĠSant os +Ġveget ation +33 8 +t ri +Ġfl ux +ins ert +Ġclar ified +Ġmort g +ĠCh im +ĠT ort +Ġdiscl aim +met al +ĠAs ide +Ġindu ction +Ġinf l +Ġathe ists +amp h +Ġe ther +ĠV ital +ĠBu ilt +M ind +Ġweapon ry +S ET +Ġ18 6 +ad min +g am +cont ract +af a +Ġderiv atives +Ġsn acks +Ġch urn +E conom +Ġca pped +ĠUnder standing +ĠH ers +ĠI z +Ġd uct +I ENT +augh ty +Ġâľ Ķ +ĠN P +Ġsa iling +In itialized +Ġt ed +Ġreact ors +ĠL omb +Ġcho ke +ĠW orm +Ġadm iration +Ġsw ung +ens ibly +Ġr ash +ĠGo als +ĠImport ant +Sh ot +ĠR as +Ġtrain ers +ĠB un +Work ing +Ġhar med +ĠPand ora +ĠL TE +Ġmush room +ĠCH AR +ĠF ee +ĠM oy +B orn +ol iberal +ĠMart ial +Ġgentle men +Ġling ering +Offic ial +Ġgra ffiti +ĠN ames +D er +Ġqu int +ist rate +aze era +ĠNOT ICE +ĠFlore nce +Ġpay able +Ġdep icts +ĠSpe cies +He art +âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ +Ġencl osed +Incre ases +D aily +ĠL is +Ġenact ment +ĠB acon +ĠSt eele +dem and +Ġ18 3 +Ġmouth s +Ġstr anded +Ġenhance ment +01 1 +ĠWh ats +Ġhe aled +en y +ĠR ab +Ġ3 40 +ĠLab yrinth +ro ach +ĠY osh +ĠCl ippers +Ġconcert s +Intern et +35 5 +Ġstick ers +Ġter med +ĠAx e +Ġgrand parents +Fr ance +ĠCl im +ĠU h +ul ic +Ġthr ill +cent ric +ĠOver view +ĠCond uct +Ġsubstant ive +Ġ18 2 +m ur +Ġstr ay +ĠCo ff +Ġrep etitive +ĠFor gotten +Ġqual ification +ew itness +ĠZ imbabwe +Ġsim ulated +ĠJ D +25 3 +ĠW are +Ġun sc +T imes +Ġsum mons +Ġdis connected +Ġ18 4 +ci us +ĠGu jar +od ka +Ġer ase +ĠTob acco +elect ed +Ġun cont +ĠShe pard +ĠL amp +Ġalert ed +Ġoper ative +arn a +u int +Ġneglig ence +ac ements +Ġsup ra +Ġprev ail +ĠSh ark +Ġbel ts +ãģ « +Ġt ighter +Engine ers +Ġin active +Ġexp onent +ĠWill ie +a ples +Ġhe ir +ĠH its +ian n +ĠS ays +Ġcurrent s +ĠBeng al +Ġar ist +B uffer +Ġbree ze +ĠWes ley +Col a +Ġpron oun +Ġde ed +ĠK ling +Ġof t +Ġinf lict +Ġpun ishing +Ġn m +ik u +OD UCT +01 4 +Ġsubsid y +ĠDE A +ĠHer bert +ĠJ al +B ank +Ġdef erred +Ġship ment +B ott +Ġal le +b earing +HT ML +Off line +Ġ2 13 +Ġscroll ing +Ġsc anned +ĠLib yan +ĠT OP +ch rom +d t +col umn +Psy NetMessage +Z ero +Ġtor so +0 50 +âķ IJ +Ġimp erson +ĠSchw artz +ud ic +Ġpiss ed +ĠS app +25 7 +ĠIS Ps +og l +Ġsuper vised +Ġad olescent +Ġatt ained +ĠDel ivery +ĠB unny +Ġ19 37 +Ġmini ature +Ġo s +Ġ3 70 +60 8 +ĠMour inho +Ġinn ate +Ġtem po +ĠN M +ĠFall en +00 9 +Ġprov ocative +Stream er +ĠBened ict +ĠBol she +Ġt urtle +ĠPC B +ĠEqu al +Direct or +ĠR end +Ġflu ids +Author ities +Ġcous ins +requ ency +ĠNeigh bor +s ets +sh ared +Char les +pass word +Ġg ears +Ġ2 11 +ĠHard ware +ri ka +Ġup stream +H om +Ġdisproportion ately +iv ities +Ġund efined +Ġelect rons +Ġcommem or +Event ually +Ġ> < +Ġir responsible +2 18 +ĠRe leased +ĠO VER +ĠI GN +ĠB read +st ellar +ĠS age +tt ed +dam age +ed ition +ĠPre c +Ġl ime +Ġconf inement +Ġcal orie +we apon +Ġdiff ering +ĠS ina +m ys +am d +Ġintric ate +k k +ĠP AT +ã o +st ones +lin ks +Ġr anch +Sem itic +Ġdifferent iate +ĠS inger +occup ied +Ġfort ress +c md +Ġinter ception +ĠAnk ara +Ġre pt +ĠSol itaire +Ġrem ake +p red +Ġd ared +aut ions +ĠB ACK +Run ning +Ġdebug ging +Ġgraph s +3 99 +ĠNig el +Ġb un +Ġpill ow +Ġprog ressed +fashion ed +Ġob edience +ER N +Ġrehe ars +C ell +t l +S her +Ġher ald +ĠPay ment +ĠC ory +ĠDe pt +Ġrep ent +ĠWe ak +uck land +Ġple asing +Ġshort ages +Ġjur ors +ĠK ab +q qa +Ant i +Ġw ow +ĠRC MP +Ġt sun +ĠS ic +Ġcomp rises +Ġsp ies +Ġprec inct +n u +Ġur ges +Ġtim ed +Ġstrip es +ĠB oots +Ġy en +Adv anced +Ġdisc rete +ĠArch angel +employ ment +D iff +Ġmon uments +Ġ20 9 +work er +Ġ19 6 +ĠI g +utter stock +T PS +J ac +Ġhomeless ness +Ġcomment ator +Ġrac ially +f ing +se ed +E le +ell ation +Ġeth anol +Ġpar ish +ĠD ong +ĠAw akening +Ġdev iation +ĠB earing +ĠTsu k +Ġrec ess +Ġl ymph +ĠCann abis +å ľ +ĠNEW S +Ġd ra +ĠStef an +ĠWr ong +ĠS AM +Ġloose ly +Ġinterpre ter +ĠPl ain +Go vernment +Ġbigot ry +Ġgren ades +ave z +pict ured +Ġmand ated +ĠMon k +ĠPed ro +Ġl ava +27 4 +Ġcyn ical +ĠScroll s +l ocks +M p +Ġcon gregation +orn ings +ph il +ĠI bid +Ġf erv +Ġdisapp earing +Ġarrog ant +sy n +ĠMa ver +ĠSu it +24 1 +Ġab bre +ack ers +P a +ĠY el +Whe never +Ġ23 5 +ĠV ine +ĠAn at +Ġext inct +LE T +Ġexecut able +V ERS +ox ide +D NA +ĠP rel +Ġresent ment +Ġcompr ise +ĠAv iv +Ġinter ceptions +Ġprol ific +IN A +ĠEr in +though t +2 19 +ĠPsychiat ry +un ky +chem ist +H o +ĠMcC oy +Ġbr icks +L os +ri ly +ĠUS SR +Ġr ud +Ġl aud +ĠW ise +ĠEmer ald +Ġrev ived +Ġdam ned +ĠRep air +id em +ct ica +Ġpatri arch +ĠN urs +me g +Ġcheap est +re ements +empt y +ĠCele br +Ġdepri vation +ch anted +ĠTh umbnails +E nergy +ĠEth an +ĠQ ing +Ġopp oses +W IND +v ik +ĠM au +ĠS UB +66 7 +G RE +ĠVol unte +nt on +C ook +å IJ +es que +Ġplum met +Ġsu ing +Ġpron ounce +Ġresist ing +ĠF ishing +ĠTri als +Ġy ell +Ġ3 10 +Ġin duct +Ġpersonal ized +oft en +R eb +EM BER +Ġview point +Ġexist ential +() ) +rem ove +MENT S +l asses +Ġev apor +Ġa isle +met a +Ġreflect ive +Ġentit lement +Ġdev ised +mus ic +asc ade +Ġwind ing +off set +Ġaccess ibility +ke red +Bet ter +ĠJohn ston +th inking +S now +ĠCroat ia +ĠAt omic +27 1 +34 8 +Ġtext book +ĠSix th +Ġ اÙĦ +Ġsl ider +ĠBur ger +b ol +S ync +Ġgrand children +Ġc erv ++ ) +Ġe ternity +Ġtweet ing +Ġspec ulative +Ġpiv otal +ĠW P +ĠT ER +ynam ic +Ġu pl +ĠC ats +per haps +Ġclass mates +Ġblat ant +' - +Ġl akh +ant ine +ĠB org +i om +/ ( +ĠAthlet ic +Ġs ar +OT A +ĠHoff man +Never theless +Ġad orable +Ġspawn ed +Ass ociated +ĠDom estic +Ġimpl ant +ĠLux em +ĠK ens +Ġp umps +ĠS AT +Att ributes +50 9 +av our +Ġcentral ized +ĠT N +Ġfresh ly +ĠA chieve +Ġouts iders +her ty +ĠRe e +ĠT owers +ĠD art +ak able +Ġm p +ĠHeaven ly +Ġr ipe +ĠCarol ine +ry an +Ġclass ics +Ġret iring +Ġ2 28 +Ġa h +Ġdeal ings +Ġpunch ing +ĠChap man +O ptions +max well +vol ume +Ġst al +Ġex ported +ĠQu ite +Ġnumer ical +B urn +F act +ĠKey stone +Ġtrend ing +Ġalter ing +ĠAfric ans +47 8 +ĠM N +ĠKn ock +Ġtempt ation +Ġprest ige +Over view +ĠTrad itional +ĠBah rain +Priv ate +ĠH OU +Ġbar r +ĠT at +C ube +US D +ĠGrand e +ĠG at +ĠFl o +Ġres ides +Ġind ec +vol ent +Ġperpet ual +ub es +Ġworld view +ĠQuant um +Ġfil tered +Ġen su +orget own +ERS ON +ĠM ild +37 9 +OT T +à ¥ +Ġvit amins +Ġrib bon +Ġsincere ly +ĠH in +Ġeight een +Ġcontradict ory +Ġgl aring +Ġexpect ancy +Ġcons pir +Ġmon strous +Ġ3 80 +re ci +Ġhand ic +Ġpump ed +Ġindic ative +Ġr app +Ġav ail +ĠLEG O +ĠMar ijuana +19 85 +ert on +Ġtwent ieth +################ ################ +ĠSw amp +Ġval uation +Ġaffili ates +adjust ed +ĠFac ility +26 2 +Ġenz ymes +itud inal +Ġimp rint +S ite +Ġinstall er +ĠT RA +m ology +lin ear +ĠCollect ive +ig ating +ĠT oken +Ġspec ulated +K N +ĠC ly +or ity +Ġdef er +Ġinspect ors +appro ved +R M +ĠSun s +Ġinform ing +ĠSy racuse +ib li +7 65 +Ġgl ove +Ġauthor ize +â̦â̦â̦â̦ â̦â̦â̦â̦ +ĠCru ise +Ġcontract ing +she ll +IF E +ĠJew el +p ract +ĠPhot oshop +ĠKnow ing +h arm +Ġattract ions +ad an +et us +01 8 +w agen +Al t +Ġmultip ly +Ġequ ilibrium +: { +ĠF ighters +ĠEd gar +Ġfour teen +Go vern +Ġmis use +Ġab using +Ġancest ry +ram er +64 4 +Ġwor ms +Ġthick er +ĠComb ine +Ġpeas ants +Ġv ind +Ġcon quest +Ġm ocked +Ġc innamon +ĠC ald +ĠGall up +Ġavoid ance +Ġincarn ation +ĠStr at +Ġt asted +ent a +ĠN eal +p ared +Ġtermin ology +ject ion +Scient ists +ĠIN S +ĠDe e +Ġdirect ories +R oad +ĠSh ap +br ight +ĠDirect ors +ĠCol umn +Ġb ob +Ġprefer ably +Ġgl itch +f urt +Ġe g +id is +C BC +Ġsur rendered +Ġtest ament +33 6 +ug gest +ĠN il +an other +Ġpat hetic +ĠDon na +Ġ2 18 +ĠA very +Ġwhis key +Ġf ixture +ĠCon quest +Ġbet s +O cc +ĠLe icester +] ." +Ġ) ); +Ġfl ashes +45 6 +Ġmask ed +ge bra +Ġcomput ed +che l +aud er +Ġdefe ats +ĠLiber ation +ĠOs ama +ĠV ive +Ch anges +Ch annel +Ġtar iffs +Ġm age +ĠS ax +Ġinadvert ently +ĠC RE +ĠRe aper +ink y +gr ading +Ġstere otyp +Ġcur l +ĠF ANT +Ġfram eworks +M om +ĠAn ch +Ġflav our +car bon +Ġperm itting +let cher +ĠMo zilla +ĠPark ing +ĠCh amp +Sc roll +Ġmurd erer +Ġrest ed +Ġow es +ĠP oss +AD D +IF F +res olution +ĠMin ing +Ġcompar ative +D im +Ġneighbour ing +ĠA ST +ĠT oxic +Ġbi ases +Ġgun fire +ur ous +ĠMom ent +19 83 +Ġper vasive +tt p +ĠNorm ally +r ir +S arah +ĠAlb any +Ġun sett +ĠS MS +ip ers +l ayer +ĠWh ites +up le +Ġtur bo +ĠLe eds +Ġthat s +ĠMin er +M ER +ĠRe ign +Ġper me +ĠBl itz +Ġ19 34 +Ġintimid ating +t ube +Ġecc entric +ab olic +box es +ĠAssoci ates +v otes +Ġsim ulate +um bo +aster y +Ġship ments +FF FF +an th +Ġseason ed +Ġexperiment ation +âĸ ł +law s +Me et +idd les +ant ics +R ating +IS IS +h ift +Ġfront s +b uf +01 7 +Ġun att +ĠD il +le ases +ĠGard ens +77 7 +t ouch +ve ll +45 8 +Ġ= ==== +s aving +Ġer osion +ĠQu in +Ġearn s +Ġaccomplish ment +ĠWe i +Ġ< [ +____ _ +Ġir rig +ĠT eddy +Ġconqu ered +ĠArm ored +Ġassert s +Ġmanip ulating +r é +Ġtranscript s +G allery +Ġplot ting +Ne il +Ġbetray al +load er +ĠS ul +Ġdispl acement +Ġroy alty +ĠW I +he it +ĠDev ices +alle l +Ġmunicipal ities +Ġcan al +St ars +ĠU AE +Ġ" â̦ +ĠC U +ab ove +Ġreson ance +ĠguiActive Un +add ed +ĠBra ves +ĠI bn +Ġhere by +ĠB RE +Ġshare holder +ĠH ir +ĠJ i +Ġstrange ly +Ġadm ired +Ġpl ight +Ġb achelor +ĠP ole +cipl inary +T ony +ĠArmen ian +Ġun man +ĠZion ist +St age +isco ver +Ġautom otive +Ġs idelines +Ġsl ick +ĠRena issance +ĠF UN +Im ages +ĠH aj +Ġp ing +Ġshort cut +ĠBl vd +ĠLook s +Ġbur sts +Ġcl amp +Ġm ish +Ġsort ing +Ġpatri ot +Ġcorrect ness +ĠScand inav +ĠCaval iers +p ython +az ar +Ġ3 75 +ĠJa une +40 9 +Ġdetrim ental +Ġstab bing +Ġpoison ed +Ġf ountain +oc ent +or st +ĠMar i +Ġr ains +ĠO vers +ĠInst itution +ud get +AM Y +t ale +ĠK R +ĠPr ices +Ġhead aches +Ġlands l +ĠA ura +Bon us +ĠZ hao +ĠH ip +Ġhop s +ĠKurd istan +Ġexplo iting +ry n +Ġhypocr isy +op ening +Ġgun shot +Ġw ed +inter stitial +Inter stitial +Ġam en +Bre aking +Ġmarket ed +W ire +ĠC rowd +Contin ue +ĠK nown +ĠEffect ive +ore an +iz ons +Jose ph +Ġescal ation +us ername +Ġcur tain +AT ES +ĠP AR +ĠM iy +Ġcounter fe +l ene +Ġcont enders +d aily +ĠAs c +ĠPhill ip +most ly +Ġfil ename +he ne +Ġresemb ling +Ġst aging +ĠCh loe +Ġw iring +H on +ĠRen ew +ott age +ĠHy brid +m uch +Ġstro kes +Ġpolicy makers +AP TER +ĠArk ham +pl ot +Ġassist ants +Ġde port +ĠSe ga +Ġinflu enza +ĠC ursed +ĠK obe +Ġskin ny +Prov ider +ĠR ip +Ġincrement al +product s +B F +Ġd ome +ĠC redits +Ġlos ers +int s +ĠBet ty +ĠTal ent +ĠD AM +L v +E ss +Ġd ens +tem p +J udge +od ic +Ġ' ( +UR ES +ets k +V O +Ġretrie ved +Ġarchitect s +Ù ĩ +Ġeth ic +ĠSecond ary +st ocks +ad ia +Ġ3 25 +ĠOp inion +Ġsimultane ous +Ġd izz +ul p +Ġsmugg ling +ipp ery +R andom +f acing +ĠD as +Ġstock p +Ġdiscl osures +po inter +Ġcor al +ĠSe lection +ĠP ike +ival ent +Ġruth less +ĠR im +Ġensu ing +ĠExper iment +Ġcongress man +Ġbelie ver +Ġun specified +ĠM ord +Ġknowledge able +ĠV ERY +T X +Ġstra ps +Ġtur f +apesh ifter +Ġmar ital +Ġfl ock +ãģ Ĩ +26 3 +AM ES +ĠOpp osition +Ġtre asures +ĠG OD +Ġmodel ed +ĠWOR LD +Ġ( [ +ĠUs age +H F +Ġ$ ( +uss ed +Ġpione er +E ight +par se +b read +rit z +ĠMir anda +ĠK ant +++ ) +ore n +Ġprov oked +Ġbre eds +ĠIn cludes +ĠPast ebin +ĠFl ip +J ava +Ġbr ink +Ġrum ored +Ġun seen +Ġgar nered +ĠDef in +al ted +Ġtatt oos +Ġhes itation +is itions +ĠWe aver +ĠReport ing +Ġtherap ies +Ġconsult ants +Ġresid ual +ĠMal i +ĠRom a +i ago +ĠRes idents +ub i +Ġremed ies +Ġadapt ive +ĠAl ive +ĠBar cl +Ġwal lets +c rypt +etermin ation +ĠPel osi +Ġsl ipping +oton in +Ġall iances +pat rick +ir is +Ġor th +ĠPer kins +ĠDe V +ĠG ets +Ġdry ing +ge e +fore st +ĠFor get +ore m +33 9 +Ġvague ly +ĠD ion +ĠP orn +ĠH OW +Ġp neum +Ġrub ble +ĠT aste +enc ia +ĠG el +Ġd st +Ġ24 5 +ĠMoroc co +inf lamm +ĠTw ins +Ġb ots +d aughter +ĠB alk +Ġbre thren +Ġlog os +Ġgo bl +f ps +Ġsub division +Ġp awn +Ġsquee zed +Ġmor ale +ĠD W +' " +Ġkn ot +ook y +Ġdiv isive +Ġboost ed +ch y +ãĥ IJ +if act +Ġnewcom ers +ĠWrest ling +Ġsc outs +w olves +R at +Ġnin eteenth +ĠOs borne +St ats +Ġem powered +Ġpsych opath +ĠO EM +ugg age +ĠP K +ĠMoh ammad +P ak +Ġanarch ists +ĠExt ract +est hes +ĠStock holm +l oo +ĠG raph +Ġdeploy ing +ĠStr anger +ĠM old +Ġstaff er +Ġdiscount ed +uck le +ple ase +ĠLand ing +ÃŃ a +Ġ19 3 +Ġan te +Ġrep etition +Ġ+ /- +Ġpar ody +Ġlive ly +AA A +ĠHor us +Ġp its +ind ers +L OC +ĠVen ice +40 6 +ĠDis cover +â Ĩ +ellect ual +Ġp ens +Ġey el +ig uous +Im pl +Ġj oking +Ġinv al +ĠBel fast +Ġcredit ors +ĠSky walker +ov sky +Ġcease fire +Ġse als +is oft +) ). +ĠFel ix +IT S +Ġt resp +ĠBlock chain +ew are +ĠSch war +en ne +mount ed +ĠBe acon +les h +Ġimmense ly +Ġche ering +Em ploy +sc ene +ish ly +atche wan +ĠNic olas +Ġdr ained +ĠEx it +ĠAz erb +j un +Ġflo ated +u ania +De ep +Ġsuper v +Ġmyst ical +ĠD ollar +ĠApost le +ĠR EL +ĠProv ided +ĠB ucks +ãĥ ´ +cut ting +Ġenhance ments +ĠPengu ins +ĠIsa iah +Ġj erk +ĠW yn +Ġst alled +Ġcryptoc urrencies +ĠR oland +sing le +Ġl umin +ĠF ellow +ĠCap acity +ĠKaz akh +W N +Ġfin anced +38 9 +Ġt id +Ġcoll usion +ĠMy r +î Ģ +Sen ator +Ġped iatric +Ġneat ly +Ġsandwic hes +ĠArchitect ure +Ġt ucked +Ġbalcon y +Ġearthqu akes +qu ire +F uture +Ġhe fty +é Ĺ +Ġspecial izes +Ġstress es +Ġs ender +Ġmisunder standing +Ġep ile +Ġprov oke +ĠCol ors +Ġdis may +uk o +[ _ +58 6 +ne utral +Ġdon ating +ĠRand all +Mult i +Ġconvenient ly +ĠS ung +ĠC oca +Ġt ents +ĠAc celer +Ġpart nered +27 2 +ir ming +ĠB AS +s ometimes +Ġobject ed +ub ric +p osed +LC S +gr ass +Ġattribut able +V IS +Israel i +Ġrepe ats +ĠR M +v ag +ut a +in ous +Ġin ert +ĠMig uel +æ Ń +ĠHawai ian +B oard +Ġart ific +ĠAzerb ai +as io +ĠR ent +A IN +Ġappl iances +Ġnational ity +Ġass hole +ĠN eb +Ġnot ch +h ani +ĠBr ide +Av ailability +Ġintercept ed +Ġcontin ental +Ġsw elling +ĠPers pect +b ies +. < +ith metic +ĠL ara +Ġtempt ing +add r +Ġoversee ing +cl ad +ĠD V +ĠGing rich +Ġm un +ĠApp ropri +Ġalter ations +ĠPat reon +Ġha voc +Ġdiscipl ines +Ġnotor iously +aku ya +ier i +? ). +ĠW ent +Ġsil icon +Ġtre mb +Cont ainer +K nown +Ġmort ar +est e +ick a +Ar thur +ĠPre viously +ĠMart y +Ġsp arse +g ins +Ġin ward +ĠParticip ant +C opy +ĠM isc +Ġantib iotic +ĠRet ro +Ġel usive +Ġass ail +ĠBatt alion +ĠB ought +Ġdimin ish +ĠEuro pa +s ession +ĠDanger ous +ies el +Ġdisbel ief +Ġbl asts +ext reme +ĠBoy d +ĠProject s +ĠGu ys +Ġunder gone +Ġgr ill +ĠDw ight +Ġ19 7 +US ER +Ġfiles ystem +Ġcl ocks +T aylor +Ġwra pper +Ġfold ing +ous and +ĠPhilipp ine +ATION AL +ĠPer th +Ġas hes +Ġaccum ulate +ĠGate way +Sh op +orks hire +H an +ĠBar rel +ĠLe h +ĠX V +Ġwh im +Ġrep o +ĠC G +ĠM am +Ġincorpor ating +Ġbail out +Ġlingu istic +Ġdis integ +C LE +Ġcinem atic +ĠF iber +S yn +il ion +ĠCom pos +c hens +Ġne oc +Ġbo iled +F INE +on o +un cle +ik en +ĠB M +Î ¹ +Ġreceipt s +Ġdisp osed +ĠTh irty +ĠR ough +ĠA BS +Ġnot withstanding +oll en +# $ +Ġunrel iable +Ġbl oom +Ġmedi ocre +Ġtr am +ĠTas man +Ġsh akes +Ġmanifest o +ĠM W +Ġsatisf actory +Ġsh ores +Ġcomput ation +Ġassert ions +orm ons +ar ag +ab it +Dem ocrats +ĠL oot +ĠVol ks +ha ired +Ġgrav itational +S ing +ĠM iz +Ġthro ttle +Ġtyr anny +ĠView s +Ġrob ber +ĠMinor ity +Ġsh rine +sc ope +pur pose +Ġnucle us +our cing +ĠUS DA +ĠD HS +w ra +ĠBow ie +Sc ale +ĠB EL +x i +I ter +Ġ( ), +w right +Ġsail ors +ous ed +NAS A +ĠPro of +ĠMin eral +t oken +ĠF D +R ew +Ġe ll +6 30 +Ġchance llor +ĠG os +Ġamount ed +ĠRec re +ome z +ĠOpt im +ĠOl ive +Ġtrack er +ow ler +ĠUn ique +R oot +Ġmar itime +ĠQur an +ĠAd apt +Ġecosystem s +ĠRe peat +ĠS oy +ĠI MP +Ġgrad uating +and em +P ur +ĠRes et +ĠTr ick +ĠPh illy +ĠT ue +ĠMalays ian +Ġclim ax +Ġb ury +Ġcons pic +ĠSouth ampton +ĠFl owers +Ġesc orted +ĠEduc ational +ĠI RC +Ġbrut ally +e ating +Ġpill ar +ĠS ang +ĠJ ude +ar ling +ĠAm nesty +Ġrem inding +ĠAdminist rative +hes da +Ġfl ashed +ĠP BS +per ate +fe ature +Ġsw ipe +Ġgra ves +oult ry +26 1 +bre aks +ĠGu er +Ġsh rimp +ĠV oting +qu ist +Ġanaly tical +Ġtables poons +ĠS OU +Ġresear ched +Ġdisrupt ed +Ġj our +Ġrepl ica +Ġcart oons +b ians +} ) +c opy +G ot +ou ched +P UT +Ġsw arm +not ations +s aid +Ġreb uilt +Ġcollabor ate +Ġr aging +Ġn ar +Ġdem ographics +ĠD DR +Ġdist rust +oss ier +ĠK ro +Ġpump kin +Ġreg rets +Ġfatal ities +ĠL ens +ĠO le +p d +Ġpupp et +ĠOut look +ĠSt am +O l +F air +U U +Ġre written +Ä ± +Ġfasc inated +Ġve ctors +Ġtrib unal +u ay +ĠM ats +ĠCo ins +[ [ +Ġ18 1 +Ġrend ers +ĠK aepernick +Ġesp ionage +Ġsum m +Ġd itch +Acc ount +Ġspread sheet +Ġmut ant +p ast +40 7 +Ġd ye +Ġinit iation +Ġ4 000 +Ġpunish able +Ġth inner +ĠKh al +Ġinter medi +D un +ĠGoth am +Ġeager ly +Ġvag inal +p owers +V W +ĠWATCH ED +Ġpred ator +ams ung +Ġdispar ity +Ġ[ * +Ġam ph +Ġout skirts +ĠSpir its +Ġskelet al +Ð » +ĠR ear +Ġissu ance +ĠLog ic +re leased +Z Z +ĠB ound +Ent ry +Ġex its +is ol +ĠFound er +Ġw re +ĠGreen land +ĠM MO +t aker +IN C +ãģ ¾ +Ġhour ly +hen ko +Ġfantas ies +Ġdis ob +Ġdemol ition +ãĥ ĭ +Ġen listed +rat ulations +Ġmis guided +Ġens ured +Ġdiscour aged +m ort +Ġfl ank +Ġc ess +Ġreact s +ĠS ere +s ensitive +ĠSer pent +ass ad +Ġ24 7 +Ġcalm ly +b usters +Ġble ed +ĠSt ro +Ġamuse ment +ĠAntar ctica +Ġs cept +ĠG aw +a q +ason ic +Ġsp rawling +n ative +atur ated +ĠBattle field +IV ERS +E B +ĠG ems +ĠNorth western +ĠFil ms +ĠAut omatic +Ġappre hend +ãģ ¨ +Ġgui Name +Ġback end +Ġevid enced +ge ant +01 2 +ĠS iege +Ġexternal To +Ġunfocused Range +ĠguiActiveUn focused +Ġgui Icon +ĠexternalTo EVA +ĠexternalToEVA Only +F ri +ch ard +en aries +Ġchief s +Ġc f +ĠH UD +Ġcorro bor +Ġd B +ĠT aken +ĠPat ricia +ra il +ĠCh arm +ĠLiber tarian +rie ve +Person al +ĠO UR +ger ies +Ġdump ing +Ġneurolog ical +it imate +ĠClint ons +raft ed +ĠM olly +Ġtermin als +reg ister +Ġfl are +Ġenc oded +Ġautop sy +p el +m achine +Ġexempt ions +ĠRoy als +d istance +Ġdraft s +Ġl ame +ĠC unning +Ġsp ouses +ĠMark ets +ĠCar rier +Ġimp lying +ĠY ak +s id +Ġl oser +Ġvigil ant +Ġimpe achment +Ġaug mented +ĠEmploy ees +Ġunint ended +tern ally +ĠW att +Ġrecogn izable +ess im +æ Ŀ +Ġco ated +r ha +Ġlie utenant +ĠLegisl ation +pub lished +44 4 +01 3 +Ġide ally +ĠPass word +Ġsimpl ify +ĠMet a +ĠM RI +Ġple ading +organ ized +hand ler +Ġun ravel +cor rect +Ġ icy +Ġparan oid +Ġpass er +Ġinspect ions +of er +ĠHealth care +28 3 +ĠBr ut +iol a +for ge +ĠMed ieval +MS N +ie vers +ĠProgram ming +å ī +Ġ2 23 +m u +ĠC LE +ug a +Ġsho ppers +Ġinform ative +ĠPl ans +Ġsupplement ation +ĠT ests +ty ard +ocy tes +ĠVeg a +ĠGujar at +erman ent +Ex cept +ĠL OT +all a +ĠC umm +ĠO sw +Ġven om +ĠDeb t +ĠD OWN +Ġreun ion +Ġm uc +ĠRel ief +Ġge op +ĠðŁ ĺ +al ogue +An th +ech o +Ġcor ros +Ġrepl ication +ĠBl azing +ĠD aughter +Ġinf lic +ĠLind sey +Ù Ī +28 4 +Ex it +Ġgl oom +TA IN +Ġundermin ing +Ġadv ising +h idden +Ġover flow +Ġg or +urd ue +Ġe choes +enh agen +Ġimp uls +d rug +c ash +Ġas ync +Ġmir ac +at ts +p unk +Ġpiv ot +ĠLegisl ative +Ġblog gers +ĠCl aw +s burg +d yl +ĠRecomm end +Ġver te +Ġprohib iting +ĠPant her +Jon athan +Ġo min +Ġhate ful +28 1 +ĠOr che +ĠMurd och +down s +Ġas ymm +G ER +Al ways +Ġinform s +ĠW M +ĠP ony +ĠApp endix +ĠAr lington +J am +Ġmedic inal +ĠS lam +IT IES +Ġre aff +ĠR i +F G +S pring +b ool +Ġthigh s +Ġmark ings +ĠRa qqa +ĠL ak +p oll +ts ky +ĠMort y +ĠDef inition +Ġdeb unk +end ered +ĠLe one +a vers +Ġmortg ages +App arently +N ic +ha us +ĠTh ousands +au ld +Ġm ash +sh oot +Ġdi arr +Ġconscious ly +H ero +e as +ĠN aturally +ĠDestroy er +Ġdash board +serv ices +R og +Ġmillenn ials +Ġinv ade +- ( +Ġcomm issions +ĠA uckland +Ġbroadcast s +Ġfront al +Ġcr ank +ĠHist oric +Ġrum ours +CT V +Ġster il +Ġboost er +rock et +ãĤ ¼ +ut sche +ĠP I +Ġ2 33 +ĠProdu cer +ĠAnaly tics +Ġinval uable +Ġunint ention +ĠC Y +Ġscrut in +Ġg igg +Ġeng ulf +Ġprolet ariat +Ġh acks +ĠH ew +ar ak +ĠSl ime +ield ing +ag her +ĠEll iot +Ġtele com +Ġ2 19 +ult an +ĠAr bor +ĠSc outs +B an +Ġlifes pan +Ġbl asp +38 8 +Ġjud iciary +ĠContin ental +ask ing +Mc C +L ED +Ġbag gage +ĠSorce rer +Ġrem nants +ĠGriff ith +ets u +ĠSub aru +ĠPerson ality +des igned +ush ima +agn ar +Ġrec oil +Ġpass ions +\ ": +Ġte e +Ġabol ition +ĠCreat ing +j ac +Ġ19 4 +01 9 +Ġpill ars +ric hed +/ " +t k +Ġlive lihood +Ġro asted +ah on +ĠH utch +ass ert +Ġdivid end +Ġkn it +Ġd aunting +Ġdisturb ance +Ġsh ale +Ġcultiv ated +Ġrefriger ator +L B +ĠN ET +Ġcommercial s +Ġthink ers +45 5 +Ġch op +B road +Ġsuspic ions +Ġtag ged +l ifting +Ġsty lish +ĠShield s +Short ly +Ġt ails +A uth +ST E +ĠG AME +Ġse ism +ĠK is +olog ne +Ġcow ork +Ġforc ibly +Ġthy roid +ĠP B +AN E +mar ried +h orse +Ġpoly mer +ĠCh al +od or +DE BUG +ĠCon text +Ġbl iss +Ġpin point +ĠMat hemat +leg ram +ĠWeek end +Ġlab elled +Ġb art +it les +Ġest rogen +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +" ' +Ġvis ibly +Ġouts ider +aid a +Are a +Ġdisse min +Ġdish onest +ĠCl osed +ĠBullet in +ĠRam sey +sw ord +ĠX I +our ced +S ame +34 6 +ĠRe pe +ĠK ou +c ake +em is +C ache +ĠMe aning +ĠEn light +onom y +Ġmanifest ation +sw orth +J ay +Ġch ore +ö r +D ream +Ġsanction ed +Ġcult urally +ĠA ra +N av +Ġthe ological +Ġstr ut +ĠV O +ĠHand book +Ġconstruct ing +Ġ ¶ +ĠBenef its +ĠPsych ological +s ac +å ¸ +p olicy +ĠMat ters +ĠReport ed +ĠBy te +Ġvit ro +ĠM aiden +Ġl am +ĠJenn ings +Ġgar ment +ĠRut gers +ĠStaff ord +ĠWell ington +Ġinter mitt +Ġn pm +Ġord eal +Ġplug ged +o oming +in ished +fram ework +Ġtim ber +Ġc ass +Ġ8 50 +il ess +ĠRed ux +7 68 +St re +Ġsurpass ed +w hel +Ġparalle ls +Ġve il +ĠG I +ĠR EST +Ġread iness +s ort +Ġmod ifying +ĠSl ate +ru ff +Ġmar ble +Ġinf rared +Ġaud itor +ĠFANT ASY +ĠP overty +ĠS PD +Ġ" ( +K y +RA Y +Ġexecut ions +ĠBever ly +ĠMarx ism +ĠBur st +ĠK ali +est ones +Clear ly +E ll +ãģ § +ĠProceed ings +T oken +IF IC +ñ a +Cent ral +ĠH aley +ĠD rama +Ġform ations +OR N +Book s +Ġdom inating +ĠFly ers +ĠCompan ion +Ġdiscipl ined +ĠYug oslav +ĠSpell s +Ġv engeance +Ġland lords +L en +ĠO gre +ano ia +Ġpier cing +Ġcon greg +Ġscore r +ob ia +Ġnic kel +ĠLear ns +Ġre jo +Ġmaster piece +Fl ash +Ġinhab ited +ĠOpen GL +ĠD ud +ĠI CO +Ġar ter +Ġpl ur +Ġmaster y +Ġlong standing +st ed +Ġw ines +Ġtelev ised +ĠSh rine +ĠBay ern +Ġâ ĵĺ +Ġencl osure +j ohn +Ġprophe ts +ĠRes urrection +ĠOrd ers +Ġun even +r als +Ġd wind +ĠL ah +ĠSl oven +37 8 +Ġins istence +aff le +ĠCl one +Ġhard ship +ĠCongress man +Ġple ad +Ġreview ers +Ġc ured +Ġ19 35 +as ley +f ake +ĠTh inking +yd ia +P ART +ĠD ota +o it +Ġwh ipped +Ġb ouncing +ĠHispan ics +com ings +Ġcann abin +ĠCh ambers +ĠZ ack +Option al +Ġco ats +Ġprow ess +ĠNort on +Ġplain ly +Ġfre ight +Ġinhib ition +Ġcl am +Ġ30 3 +ke f +ale igh +L uke +Ġpsych o +ator ium +M ED +Ġtreat ies +Ġind isc +Ġd c +OP S +Ġresil ient +ĠInter state +Ġsl ack +Ġmund ane +Ġestab lishes +35 9 +Ġstr ained +Ġn ond +S us +Ġcast e +ar ate +ie ving +Ġunfair ly +Ġpars er +on ial +urs ive +V ia +ĠOtt o +ĠAuthor ities +stro ke +K R +ĠMer cy +Ġfurn ished +Ġout set +Ġmet ic +19 82 +olith ic +ĠT ent +og ical +ĠA ircraft +Ġh ides +ĠBec ame +Ġeduc ators +re aching +Ġvol atility +Ġtodd ler +ĠNAS CAR +ĠTw elve +ĠHigh lights +Ġgra pe +Ġspl its +Ġpe asant +Ġre neg +ĠMS I +Tem p +st ars +Ġtre k +ĠHy de +b inding +Ġreal ism +Ġox ide +ĠH os +Ġmount s +Ġbit ing +Ġcollaps ing +Ġpost al +Ġmuse ums +Ġdet ached +Ġrespect ing +Ġmonop ol +Ġwork flow +ĠC ake +Tem plate +ĠOrgan isation +Ġpers istence +36 9 +C oming +B rad +Ġredund ant +ĠG TA +Ġb ending +Ġrev oked +Ġoff ending +Ġfram ing +Ġprint f +Comm un +mem bers +Out side +Ġconst rued +Ġc oded +F ORE +Ġch ast +Ch at +Ind ian +ĠY ard +? !" +ĠP orts +ĠX avier +ĠR ET +' ." +ĠBo at +iv ated +ich t +umer able +D s +ĠDun n +Ġcoff in +Ġsecure ly +ĠRapt ors +ĠB es +Install ation +Ġin ception +ĠHealth y +end ants +Ġpsych ologists +ĠShe ikh +c ultural +ĠBlack Berry +sh ift +F red +oc he +Ġc akes +ĠS EO +ĠG ian +ĠAs ians +og ging +e lement +Ġpund its +ĠV augh +ĠG avin +Ġh itter +Ġdrown ed +Ġch alk +ĠZ ika +Ġmeas les +80 2 +â̦ .. +ĠAW S +] " +Ġdist ort +ĠM ast +Ġantib odies +ĠM ash +Mem ory +ĠUg anda +ĠPro b +Ġvom iting +ĠTurn s +Ġoccup ying +Ġev asion +ĠTher apy +Ġprom o +Ġelect r +Ġblue print +ĠD re +pr iced +ĠDep ot +Ġallev iate +ĠSom ali +m arg +n ine +Ġnostalg ia +ĠShe pherd +Ġcaval ry +Ġtor ped +ĠBlood y +x b +Ġs ank +Ġgo alt +report print +embed reportprint +clone embedreportprint +ĠIn itially +ĠF ischer +Ġnot eworthy +c ern +Ġin efficient +raw download +rawdownload cloneembedreportprint +c ation +ĠD ynasty +l ag +D ES +Ġdistinct ly +ĠEston ia +Ġopen ness +Ġg ossip +ru ck +W idth +ĠIb rahim +Ġpet roleum +Ġav atar +ĠH ed +ath a +ĠHog warts +Ġc aves +67 8 +Ġsafegu ard +ĠM og +iss on +ĠDur ham +sl aught +ĠGrad uate +Ġsub conscious +ĠEx cellent +ĠD um +---- - +Ġp iles +ĠW ORK +ĠG arn +ĠF ol +ĠAT M +Ġavoid s +ĠT ul +Ġble ak +EL Y +iv ist +light ly +P ers +ĠD ob +ĠL S +Ġins anity +Î µ +atal ie +En large +Ġtw ists +Ġfault y +Ġpir acy +Ġimp over +Ġrug ged +ĠF ashion +Ġs ands +' ? +sw ick +Ġn atives +Ġhe n +ĠNo ise +ãĥ Ĺ +Ġg reens +Ġfree zer +Ġd ynasty +ĠFather s +ĠNew ark +Ġarchae ological +Ġo t +ob ar +Ġblock ade +Ġall erg +L V +Ġdeb it +ĠR FC +ĠMil ton +ĠPress ure +Ġwill ingly +Ġdisproportion ate +Ġopp ressive +Ġdiamond s +Ġbelong ings +19 70 +Ġbell s +Ġimperial ism +Ġ2 27 +Ġexpl oding +ĠE clipse +Ġ19 19 +Ġr ant +Ġnom inations +34 7 +Ġpeace fully +ric a +ĠF UCK +Ġvib ration +mal ink +Ġro pes +ĠIv anka +ĠBrew ery +ĠBook er +ĠOw ens +go ers +Serv ices +ĠSn ape +Ġ19 1 +39 5 +Ġ2 99 +just ice +Ġb ri +Ġdisc s +Ġprom inently +Ġvul gar +Ġsk ipping +l ves +Ġtsun ami +37 4 +ĠU rug +ĠE id +rec ated +p hen +Ġfault s +ĠStart ed +9 50 +Ġp i +Ġdetect or +Ġbast ard +Ġvalid ated +Space Engineers +OUR CE +Ġ( ~ +Ġuns ur +Ġaff irmed +Ġfasc ism +Ġres olving +ĠCh avez +ĠC yn +Ġdet ract +L ost +Ġrig ged +Ġhom age +ĠBrun o +55 5 +ec a +Ġpress es +Ġhum our +Ġsp acing +Ġ' / +olk ien +C oun +OP ER +T re +S on +ĠCambod ia +ier re +m ong +o zy +Ġliquid ity +ĠSov iets +ĠFernand o +Ġ2 29 +Ġsl ug +ĠCatal an +elect ric +Ġsc enery +ĠH earth +Ġconst rained +Ġgoal ie +ĠGu idelines +ĠAm mo +ĠPear son +Ġtax ed +Ġfet us +Resp onse +ĠAlex is +th ia +G uy +Ġrecon struct +Ġextrem es +Ġconclud ing +ĠP eg +ook s +Ġded uctions +R ose +Ġground breaking +ĠT arg +ãĥ ģ +ĠRe ve +res ource +Ġmo ons +Ġelectrom agnetic +Ġamid st +ĠVik tor +N ESS +B ACK +Ġcomm ute +ĠAna heim +Ġfluct uations +6 40 +Ġnood les +ĠCop enhagen +ĠT ide +ĠGri zz +ĠS EE +Ġpip elines +Ġsc ars +end o +ag us +ĠE TF +/ # +ĠBec ome +44 8 +Ġvis c +ĠRecomm ended +Ġj umper +Ġcogn ition +Ġassass in +Ġwitness ing +ĠSet up +Ġl ac +v im +IS M +p ages +SS L +35 8 +Ġad ject +indust rial +l ore +cher y +Ġgl itter +Ġc alf +Flor ida +Ġspoil ers +Ġsucceed s +Ġch anting +Ġslog ans +ĠTr acy +Vis it +rol ogy +Ġm ornings +Ġline age +Ġs ip +Ġintense ly +Ġflour ish +ĠSle eping +ĠF em +or por +ĠK lan +ĠDar th +h ack +ĠNi elsen +Ġtum ors +Ġprocure ment +ĠY orkshire +Ġra ided +K Y +An na +Ġ// [ +ĠDis order +ĠMust ang +ĠW en +ĠTry ing +s q +Ġdeliver ies +Ġshut ter +Ġcere bral +Ġbip olar +ĠC N +l ass +j et +Ġdeb ating +> : +Ġe agle +gr ades +ĠD ixon +UG C +M AS +ĠDr aco +ĠMach ines +aff er +Ġem an + ² +pr on +ĠG ym +Ġcompar atively +ĠTrib unal +PR O +Ġle x +Ġfert ile +Ġdep ressing +Ġsuperf icial +ess ential +ĠHun ters +g p +Ġprom inence +L iber +ĠAn cest +ote chnology +Ġm ocking +ĠTra ff +ĸ ļ +Med ium +I raq +Ġpsychiat rist +Quant ity +ĠL ect +Ġno isy +5 20 +G Y +Ġsl apped +ĠM TV +Ġpar a +p ull +Mult iple +as her +Ġn our +ĠSe g +Spe ll +v ous +ord ial +Sen ior +ĠGold berg +ĠPl asma +ne ed +Ġmess enger +ere t +Ġteam ed +Ġliter acy +ĠLe ah +ĠD oyle +Ġem itted +U X +Ġev ade +Ġm aze +Ġwrong ly +ĠL ars +Ġstere otype +Ġpled ges +Ġarom a +ĠM ET +Ġac re +ĠO D +Ġf f +Ġbrew eries +ĠH ilton +und le +ĠK ak +ĠThank fully +ĠCan ucks +in ctions +ĠApp ears +Ġco er +Ġundermin ed +ro vers +And re +Ġbl aze +um ers +Ġfam ine +amp hetamine +ulk an +Am ount +Ġdesper ation +wik ipedia +develop ment +ĠCor inth +uss ia +Jack son +L I +N ative +R s +Oh io +ĠKath leen +F ortunately +Ġattend ant +ĠPre ferred +ĠDid n +ĠV s +M is +Ġrespond ent +Ġb oun +st able +Ġp aved +Ġunex pl +ĠChe ney +L M +ĠC ull +bl own +Ġconfront ing +oc ese +serv ing +W i +ĠLith uania +ann i +Ġst alk +h d +Ġv ener +AP H +ynchron ous +UR R +um ably +hist oric +H alf +H ay +Ġresil ience +spe ction +Ġabandon ing +O bs +ĠDeb bie +Ġgrad ient +ĠPl aint +ĠCan al +AR CH +Ġexpans ive +Ġfun g +Ġb ounced +U nd +Ġprec autions +Ġclar ification +Ġd agger +Ġgri ps +Ġ µ +ĠRiver a +ĠUnd ead +is ites +ĠFIR ST +ñ o +aud i +Ġhost ages +Ġcompl iant +Ġal umni +Se ven +Ġcyber security +e ither +Col lect +Ġinvari ably +ĠS oci +Ġlaw maker +Ġa le +ĠPerson ally +N azi +Ġcustom ization +ĠPro c +ĠSask atchewan +eat uring +Ġsp ared +Ġdiscontin ued +Ġcomput ational +ĠMotor ola +Ġsuprem acist +government al +Ġparad ise +ĠDown ing +ĠNik on +Ġcat alyst +ber ra +Tor onto +8 75 +bet a +ĠMac ron +Ġunreal istic +ve ctor +ĠVeh icles +it iveness +ĠR V +ĠCol bert +s in +o ji +ent in +ĠKr ish +hell o +ff ield +ok y +ĠT ate +Ġmap le +Ġa ids +chem ical +33 4 +n uts +ĠWar p +Ġx x +ĠRob b +umer ous +_- _ +ft ime +ĠV W +Ġw inger +ĠD ome +t ools +ĠP V +ĠGe orgetown +Ġg eared +Ġjihad ists +Ġc p +Ġster oids +M other +cler osis +ĠDR M +nes ia +Ġl inger +Ġimm ersive +ĠC OUN +Ġoutwe igh +ens ual +B and +Ġtransform s +mat ched +ps ons +ĠJud icial +f actor +Ġrefer ral +Ġodd ly +ĠW enger +B ring +ĠB ows +60 2 +IC LE +Ġl ions +ĠAcad emic +ĠTh orn +ĠRa ider +kef eller +St orage +L ower +ĠOr t +ĠEqu ality +AL T +ĠS OC +T ypes +Ġl yn +ĠAss et +co at +TP P +C VE +ĠPione er +app lication +Mod ern +ĠH K +En vironment +Al right +R ain +IP P +ĠShi ite +Ġm ound +ĠAb ilities +cond ition +St aff +Ġcompet ence +ĠM oor +ĠDi ablo +Ġwith held +Ġost ensibly +ĠB rom +Ġms g +Ġden omin +ĠRef erences +ĠF P +Ġplun ged +Ġp amph +m oving +cent ral +Ġdown right +Ġf ading +T al +T yp +ĠTh y +uk es +it he +Ġo ve +Ġbatt led +Ġseaf ood +Ġfig ur +ĠR D +c rop +Ġsqu ads +{ \ +à ¹ +ĠE h +Ġinterview ing +ĠQ in +Ġas piring +PL IC +Ġcla uses +ĠG ast +ĠN ir +Ġl uggage +Ġh ose +Ġsystem d +Ġdesc ending +ĠRev ised +ĠR ails +al ign +70 9 +33 7 +Ġf ug +charg ing +t ags +Ġut er +k ish +WAR NING +49 0 +prof its +Ġvoy age +Ġa ce +ĠV anguard +ĠT anks +ĠM uk +Ġ2 26 +S afe +Ar mor +Ġvolcan ic +Ġwom b +ĠM IL +Ġbegin ner +ĠRec ogn +ĠA AP +PL AY +) ! +Ġdetect ing +c n +Ġbre aches +Bas ically +ĠP ag +ĠMunicip al +ĠInd ie +ĠL af +ĠDis able +ĠOl son +Ġrest rained +Ġrul ings +Ġhum ane +ev ents +ĠCinem a +display Text +ĠH atch +action Date +onna issance +Ġassault ing +ĠL ug +CH AT +Ġvig orous +ĠPer se +Ġintoler ance +ĠSnap chat +ĠSh arks +Ġd ummy +ĠDi agn +ĠGu itar +im eters +40 3 +RE G +A x +Ġsepar ates +ĠMah m +Ġt v +j ah +O OL +C irc +ĠWinds or +uss ian +Ġintu ition +Ġdis dain +ĠDon ovan +Ġ2 21 +E mb +Ġcondem ning +Ġgener osity +zz y +Ġpant ies +ĠPre vent +Action Code +AN A +34 2 +external ActionCode +Ġspec ifying +Ġcryst all +J ere +Ġru pt +ĠApp rentice +Ġprof iling +Ð º +St rike +Ġsid eline +Ġoblig ated +Ġocc ult +Ġbureaucr atic +ant ically +rupt ed +neg ative +ĠEthiop ia +ĠC ivic +Ġins iders +el igible +ĠTV s +ĠB AR +ĠT I +i ologist +ĠA IR +Ġsubstit uted +Ar ab +ĠS aul +ĠY og +p rem +Ġbuild ers +Ġstation ary +Ġdoubt ful +Ġvig orously +Ġthr illing +Ph ysical +ĠCare y +ĠHyd ra +geon ing +ĠS ly +y ton +Ġborrow ers +ĠPark inson +Ġ ë +ĠJama ica +Ġsat ir +Ġinsurg ents +ĠF irm +Ġis ot +ĠK arn +our ning +ak ens +doc s +l ittle +ĠMon aco +CL ASS +Tur key +L y +ĠCon an +ass ic +Ġstar red +ĠPac ers +et ies +Ġt ipping +M oon +ĠR w +s ame +Ġcav ity +Ġgo of +ĠZ o +Sh ock +um mer +Ġemphas izes +Ġreg rett +Ġnovel ty +Ġen vy +ĠPass ive +r w +50 5 +Ġind ifferent +ĠR ica +ĠHim self +ĠFred die +Ġad ip +ä¸ Ģ +Ġbreak out +Ġhur ried +ĠHu ang +ĠD isk +Ġro aming +?????- ?????- +U V +ĠRick y +ĠS igma +Ġmarginal ized +Ġed its +Ġ30 4 +mem ory +Ġspec imen +29 3 +ãģ ¯ +Ġvert ically +Ġaud ition +ĠHe ck +Ġc aster +ĠHold ings +ad al +ĠC ron +ĠL iam +Ġdef lect +P ick +ĠDeb ug +RE F +Ġvers atility +ot hes +class ified +ĠMah ar +ĠH ort +C ounter +st asy +not iced +33 1 +ĠSh im +f uck +ĠB ie +Ġair ing +ĠPro tein +ĠHold ing +Ġspect ators +ili ated +ĠThat cher +n osis +ãĥ¼ ãĥ³ +Te le +B oston +ĠTem pl +st ay +Ġdecl arations +47 9 +Vol ume +ĠDesign er +ĠOver watch +id ae +Ġon wards +Ġn ets +ĠMan ila +part icularly +Ġpolit ic +o other +Ġport raits +Ġpave ment +c ffff +Ġs aints +Ġbegin ners +ES PN +Ġshort comings +âķIJ âķIJ +Ġcom et +ĠOrgan ic +qu el +Ġhospital ized +Bre ak +Ġpe el +dyl ib +asp x +ur ances +ĠT IM +P g +Ġread able +ĠMal ik +Ġm uzzle +Ġbench marks +d al +ĠV acc +ĠH icks +60 9 +ĠB iblical +he ng +Ġover load +ĠCivil ization +Ġimm oral +Ġf ries +ãĤ Ĵ +Ġreprodu ced +Ġform ulation +j ug +ire z +g ear +Ġco ached +Mp Server +ĠS J +ĠK w +In it +d eal +ĠO ro +ĠL oki +ĠSong s +Ġ23 2 +ĠLou ise +asion ally +Ġunc ond +olly wood +Ġprogress ives +ĠEn ough +ĠDo e +Ġwreck age +Ġbr ushed +ĠBase Type +Ġz oning +ish able +het ically +ĠC aucus +ĠH ue +Ġk arma +ĠSport ing +Ġtrad er +Ġseem ing +ĠCapt ure +4 30 +b ish +Ġt unes +Ġindo ors +ĠSp here +ĠD ancing +TER N +Ġno b +ĠG ST +m aps +Ġpe ppers +F it +Ġoverse es +ĠRabb i +ĠR uler +vert ising +off ice +xx x +Ġra ft +Ch anged +Ġtext books +L inks +ĠO mn +ãĢ ij +Ġinconven ience +ĠDon etsk += ~ +Ġimplicit ly +Ġboost s +ĠB ones +ĠBo om +Cour tesy +Ġsens ational +AN Y +Ġgre edy +ed en +Ġinex per +ĠL er +ĠV ale +Ġtight en +ĠE AR +ĠN um +Ġancest or +S ent +ĠH orde +urg ical +all ah +Ġsa p +amb a +ĠSp read +tw itch +Ġgrand son +Ġfract ure +Ġmoder ator +ĠSe venth +ĠRe verse +Ġestim ation +Cho ose +Ġpar ach +Ġbar ric +ãĢ IJ +Ġcomp ass +Ġall ergic +âĢ ķ +OT HER +err illa +Ġw agon +Ġz inc +Ġrub bed +ĠFull er +ĠLuxem bourg +ĠHoo ver +Ġli ar +ĠEven ing +ĠCob b +est eem +Ġselect or +ĠB rawl +is ance +ĠE k +Ġtro op +Ġg uts +ĠApp eal +ĠTibet an +Ġrout ines +ĠM ent +Ġsummar ized +steam apps +Ġtr anqu +Ġ19 29 +or an +ĠAut hent +Ġg maxwell +Ġappre hens +Ġpo ems +Ġsa usage +ĠWeb ster +ur us +Ġthem ed +Ġl ounge +Ġcharg er +Sp oiler +Ġsp illed +h og +ĠSu nder +ĠA in +ĠAng ry +Ġdis qual +ĠFrequ ency +ĠEther net +Ġhel per +Per cent +Ġhorr ifying +Ġa il +ĠAll an +EE E +ĠCross ing +44 9 +Ġh olog +ĠPuzz les +ĠGo es +eren n +60 4 +ãģ ı +ĠRaf ael +Ġatt en +ĠE manuel +Ġup ro +ĠSus p +P sych +ĠTr ainer +ĠN ES +ĠHun ts +bec ue +Ġcounsel or +R ule +Ġtox ins +Ġb anners +r ifice +Ġgreet ing +Ġfren zy +Ġall ocate +Ġ* ) +ex pr +50 3 +ĠCh ick +ĠT orn +Ġconsolid ation +ĠF letcher +sw itch +fr ac +cl ips +ĠMcK in +ĠLun ar +Mon th +IT CH +Ġscholar ly +rap ed +39 8 +Ġ19 10 +Ġe greg +Ġin secure +Ġvict orious +cffff cc +Ġsing led +Ġel ves +ĠW ond +bur st +Ġcam oufl +ĠBL ACK +Ġcondition ed +ç ī +ans wered +Ġcompuls ory +asc ist +Ġpodcast s +ĠFrank furt +bn b +Ġne oliberal +ĠKey board +ĠBel le +w arm +Ġtrust s +Ġins ured +ĠBu cc +us able +60 7 +ĠPl ains +Ġ18 90 +Ġsabot age +Ġlod ged +f elt +Ġg a +ĠN arc +ĠSal em +Ġsevent y +ĠBl ank +p ocket +Ġwhis per +Ġm ating +om ics +ĠSal man +ĠK ad +Ġan gered +Ġcoll isions +Ġextraord inarily +Ġcoerc ion +G host +b irds +è Ģ +k ok +Ġper missible +avor able +Ġpo inters +Ġdiss ip +ac i +Ġtheat rical +ĠCos mic +Ġforget ting +Ġfinal ized +å¤ § +y out +l ibrary +Ġbo oming +ĠBel ieve +ĠTe acher +ĠL iv +ĠGOOD MAN +ĠDomin ican +OR ED +ĠPart ies +Ġprecip itation +ĠSl ot +R oy +ĠComb ined +Ġinteg rating +Ġch rome +Ġintest inal +ĠRe bell +Ġmatch ups +Ġblock buster +ĠLore n +ĠLe vy +Ġpre aching +ĠS ending +ĠPur pose +ra x +f if +Ġauthor itative +ĠP ET +ast ical +Ġdish on +Ġchat ting +Ġ"$ :/ +Connect ion +Ġrecre ate +Ġdel inqu +Ġbro th +ĠD irty +ĠAd min +z man +Ġscholars hips +Ġ25 3 +cont act +als a +7 67 +c reen +abb age +Ġ19 15 +Ġbl ended +Ġal armed +L anguage +35 6 +Ġbl ends +ĠCh anged +W olf +Ġhe pat +Creat ing +Ġper secut +Ġsweet ness +art e +Ġforfe iture +ĠRober to +im pro +N FL +ĠMag net +Det ailed +Ġinsign ificant +ĠPOL IT +ĠBB Q +ĠC PS +Ġse aw +amin er +m L +end if +f inals +Ġ26 5 +u ish +Ġ} ) +ĠPro blems +Ġem blem +Ġserious ness +Ġpars ing +Ġsubst itution +Ġpress ured +Ġrecy cled +ale b +Rub y +Ġprof iciency +Dri ver +ĠW ester +: ' +AF TA +Ġm antle +ĠClay ton +fl ag +Ġpractition er +c overed +ĠSt ruct +add afi +4 25 +ĠTown ship +ĠHyd ro +Lou is +34 3 +Ġcond o +ĠT ao +Ġutil ization +Ġnause a +ĠDem s +rid ges +p ause +Ġform ulas +Ġchall enger +37 6 +Ġdefect ive +ĠRail way +ĠPub Med +Ġyog urt +l bs +ĠNor folk +OP E +ĠMood y +Ġdistribut or +Ġscroll s +Ġextract s +St an +Ġv iability +Ġexp oses +Ġstar vation +ĠStep s +ĠD odd +f ew +ST D +33 2 +Ġclos ures +Ġcomplement ary +ĠS asha +ump y +Ġmon et +Ġartic ulate +ĠDo ct +k iller +Ġsc rim +Ġ2 64 +Ġprost itutes +Ġse vered +Ġattach ments +Ġcool ed +L ev +ĠF alk +f ail +Ġpolic eman +ĠD ag +Ġpray ed +ĠK ernel +Ġcl ut +Ġc ath +Ġan omaly +St orm +em aker +ĠBreak fast +ul i +o ire +J J +h z +Oper ation +ĠS ick +35 4 +ĠGuatem ala +R ate +Ġexp osures +f aces +ĠArch ae +ra f +ĠM ia +Ġ20 25 +Ġop aque +Ġdisgu ised +ĠHead quarters +S ah +Ġp ots +9 78 +ĠM alf +Ġfrown ed +Ġpoison ous +ĠCon vers +ee ks +Ġcr ab +." " +Ġtre ason +Ġr anc +Ġescal ating +Ġwar r +Ġmob s +Ġl amps +ĠSun shine +ĠBrun swick +Ph ones +Ġspe lled +ĠSk ip +Ġ20 50 +Ġ19 11 +ĠPl uto +ĠAm end +Ġme ats +38 7 +Ġst omp +ĠZh ou +ĠLevi athan +ĠHaz ard +ad v +ĠOr well +Ġal oud +Ġb umper +ĠAn arch +ub untu +ĠSer ious +f itting +ĠOption al +ĠCec il +RE AM +Ġser otonin +Ġcultiv ate +ag ogue +} \ +Ġmos ques +ĠSun ny +Ġre active +rev olution +ĠL up +ĠFed ora +Ġdefense man +ĠV ID +ist ine +Ġdrown ing +ĠBroad casting +Ġthr iller +ĠS cy +Ġacceler ating +Ġdirect s +od ied +b ike +d uration +Ġpain fully +R edd +Ġproduct ions +Ġg ag +Ġwh ist +Ġs ock +Ġinf initely +ĠConc ern +ĠCit adel +Ġlie u +Ġcand les +ogene ous +arg er +Ġheaven ly +inflamm atory +Per formance +C s +ruct ose +az aki +Ġp essim +Ġinf erence +Ġpow d +ĠZ oe +Ġpain ts +Ġd azz +pt a +-------- --- +Ġins pir +ĠExper imental +ĠKn ife +reg or +b ors +Ġshow ers +rom eda +Ġs aint +Ġben ign +ĠJ iang +Ġenvision ed +Ġsh roud +IF T +H O +Ġsh uff +ĠI CC +Ġse greg +Ġrevis it +ighth ouse +L i +Ġsub strate +ĠSe as +ĠRew ard +ĠH ep +ĠBr ass +s bm +Ġelim inates +Ġst amina +ĠV AT +ĠLo an +Ġconst raint +Ġappropri ated +Ġp es +ĠA LE +r anging +Ġ40 4 +39 2 +Ġintellectual s +ach u +Ġrestruct uring +ĠLe vin +Ġrun es +Ġdelight ful +Ġcarbohyd rates +ĠMod els +ĠExp o +Ġtransport ing +all oc +Ġring ing +S amsung +Ġscarce ly +ĠURL s +ĠM AS +Ġprot otypes +Ġnarr ator +ĠCPU s +cd n +ĠBart on +Ġdecided ly +ĠSh u +ix ir +oc ious +ĠMy st +N intendo +Ġre use +Ġforg iven +F ew +in ical +n at +Ġseam less +ĠEv a +ĠE VE +ĠJ O +land ers +Ġso fter +neg ie +Ġtrans ient +Ġorb ital +Ġfulf il +ĠK om +Hop efully +Ġdynam ically +ĠHun ger +å Ľ +ĠArmen ia +el man +ber to +Ġp ige +ĠID s +lim it +Ġve ins +Ġso aring +p acks +Gold en +ĠCr ab +ist or +ĠR PM +Ġ$ $ +g ression +Ġjihad ist +Ġgam ble +Ġcare g +Ġinf lated +F ace +ĠFire arms +ĠEm manuel +â Ŀ +Ġsh ocks +gr ab +Ġspl end +ĠHP V +ab ortion +Ab ove +Ent ity +play ers +Ġcomm enced +ul ence +Ġfulfill ment +Ġembod iments +ĠW elfare +Ġha il +Ġ< @ +tt en +Ġcat cher +ĠJ azeera +Ġvolcan o +Ġstabil ize +ĠHand ler +Ġintens ified +ĠAb rams +Ġhum iliation +p aced +60 5 +ĠCent OS +Spe cific +Ġhe ed +ĠC AM +ĠGal ile +D ie +Ġabol ished +ĠThom son +ĠTe achers +ĠW ass +j ong +ĠIS BN +ĠAll ies +sh ake +å · +v ict +How ard +Ġde em +Ġexceed ingly +ĠSmart stocks +ib e +Ġdoor way +Ġcompet ed +ig mat +Ġnational ists +Ġg room +ĠKe en +Ġdispos able +de cl +ĠT olkien +ĠSche me +Ġb iod +Ġav id +ĠEl on +ag ar +ĠT SA +R oman +Ġartific ially +Ġadvis ors +X L +ĠInf erno +36 6 +Ġted ious +ĠPhot ography +ĠCar rie +Ġtro pe +ĠSand ra +Ġdec imal +Que en +ĠGund am +ĠO M +ote ch +N BA +Ġ19 32 +Ġent renched +ĠMar ion +Ġfr aternity +Lab our +Hen ry +Ġlat itude +E ither +Ġenh ances +ĠPot ential +Ġsh ines +id ad +Ġbread th +Ġcapac ities +ĠðŁ ĻĤ +ĠBron x +Ġsex es +Ġdifferent iation +Ġheavy weight +ĠT aj +d ra +Ġmigr ate +Ġexhaust ion +ĠR UN +els ius +ĠCu omo +Ġgu itars +Ġcl ones +ĠSom ew +ĠP ry +------------ - +Ġwarr anted +cy cles +Ġsalv age +Ġdis ks +R ANT +ĠNGO s +ĠMart ian +":[ {" +Ġadd icts +oj ure +il let +Ġamazing ly +art ments +p ixel +ĠGPU s +Lay out +è £ +ĠTam il +ĠBas il +Ġimpart ial +ĠSt ructure +f ork +b ryce +Ġr idge +ĠHamb urg +ri ous +Ġbl itz +cig arettes +Ġcan ned +40 2 +Ġiron ically +Ġcompassion ate +ĠHaw kins +. # +ĠCat hedral +Ġrall ied +in ternal +Ġqu ota +st akes +T EXT +m om +Ġcomple tes +Ġ23 8 +Ġsh rug +ãĥ ij +ĠN inth +Ġrev ise +ĠProv ider +Ġtre acher +Ġqu asi +ĠPR ES +Ġdep osition +Ġconfidential ity +iss ors +Ġim balance +Ġspan ning +Ġang ular +ĠC ul +commun ication +ĠNor a +ĠGen ius +op ter +Ġs acked +Sp ot +Ġfine ly +ĠCH R +28 2 +w aves +Pal est +ĠRo hing +N L +è ¿ +Ġsh itty +ĠSc alia +4 75 +Pro gress +Ġreferen cing +Ġclass rooms +ab ee +Ġs od +hes ion +70 8 +ĠZucker berg +ĠFin ish +ĠScot ia +ĠSav ior +ĠInstall ation +an tha +( - +Ġ30 2 +ĠP unk +Ġcr ater +yout u +Ġro ast +Ġinflu encing +Ġd up +ĠJ R +ĠG rav +Ġstat ure +Ġbath rooms +A side +W iki +me an +ĠZ ak +ĠOn es +ĠN ath +Ġhyper t +Ġcommence ment +C ivil +Ġmoder ately +Ġdistribut ors +Ġbreast feeding +Ġ9 80 +ĠS ik +ĠC ig +ĠAM ER +R IP +ĠCare er +ust ing +Ġmess ed +Ġe h +ĠJ ensen +/ $ +Ġblack mail +Ġconvers ions +Ġscientific ally +Ġmant ra +p aying +Ġiv ory +ĠCour ts +OU GH +aunt let +Ser ial +B row +ĠH undreds +3 23 +Ġpe e +Ġlin ux +Ġsub mer +ĠPrinc ipal +48 5 +ĠD SL +ĠCous ins +Ġdoctr ines +ĠAthlet ics +Ġ3 15 +ĠK arma +Ġatt ent +ur ger +Ġpresc ribe +Ġenc aps +ĠC ame +Ġsecret ive +ĠCr imes +d n +C lean +ĠEgypt ians +ĠCar penter +Ġ ll +H um +ĠMil o +Ġcapital ists +Ġbrief ed +T we +ĠBas in +elve t +M os +Ġplun ge +ĠKa iser +ĠFu j +ill in +Ġsafegu ards +Ġo ste +ĠOpportun ity +ĠM afia +ĠCall ing +ap a +ur ban +br ush +ill ard +c é +int elligence +ĠL ob +ĠDru id +Ġsm oother +Ġfoot ing +Ġmotor ists +arc ity +Ġmascul inity +Ġm ism +Ġabdom inal +ĠTa vern +ĠR oh +Ġesc apes +s igned +Anth ony +Ġsacrific ing +Ġintim acy +Ġan terior +ĠK od +Ġmot if +Ġg raz +Ġvisual ization +Ġguitar ist +ĠTro tsky +m agic +D ar +ĠMor i +Ġw ards +Ġtoile ts +l est +Ġtele port +ĠSund ays +ĠPl at +ET S +Ġe Sports +Pat rick +ĠK atherine +en ko +Ġhas sle +ĠM ick +gg les +Ġh ob +aint ain +Ġair borne +Ġsp ans +Ġch ili +Ġa perture +Ġvolunte ered +ĠInc ident +ĠF res +ĠVeter an +augh tered +ing o +Ġun insured +CL OSE +Ġf use +Ġer otic +Ġadvert ise +ra ising +Text ure +Ġatt ends +ĠRE AL +udd led +Ġsm oot +Ġ30 5 +ĠWill is +Ġbl ond +An alysis +ĠV T +on ica +Ġstrongh old +R F +N M +. >> +Ġprosper ous +Ġbo asted +29 2 +ĠManufact uring +PR ESS +g ren +Ġpharm acy +ĠRoc kefeller +k ai +Ġth umbs +ĠH ut +Ġmother board +Ġguard ians +ĠAl ter +ll ular +Ġsh ack +Ġwise ly +Ġback bone +erv a +Ġsu icides +ĠMcG regor +ij ah +E mer +ĠB rav +Ġdesign ate +P OST +produ ced +Ġcleans ing +irl wind +ex istent +ĠHum ph +ĠPay ne +Ġv ested +Å ¡ +Ġstring ent +ion a +Ġuns ub +Ġsum med +ĠHer cules +sub ject +ĠR agnar +ĠN os +Ġcharacter ization +Ġsav vy +ĠDaw son +ĠCas ino +Ġf ri +ĠBar rier +Ġmis information +Ġins ulation +Ġcorrid ors +Ġair planes +ĠNo ct +ah i +Ġ19 16 +k b +arm ac +Ġsh un +Ġsche ma +Ġhorr ified +Ġ23 9 +aund ers +N B +i ates +er ity +ĠSh ard +Ġr arity +Ġgroup ed +ĠGh ana +again st +ĠBi ological +ĠA ware +ow ell +Ï Ħ +ĠBe au +sh aw +H ack +ĠJul ius +US S +ol son +aun a +c ru +ĠMaur ice +ĠI k +Ġsequ encing +Ġradical s +Ġ( ?, +v irtual +Ġany ways +Ġreper c +Ġhand lers +Ġhes itant +é ĥ +ĠM F +ple mentation +ass ociated +Ġcampaign ed +ĠY ue +ut ations +ĠY oga +Ġsim mer +Ġro ds +Ġmel ody +Ġconv oy +v ideos +Ġscreen ed +N eg +ochem ical +Ġ( )) +Ġultr as +Ġant ip +ĠIsland ers +70 4 +Ġfet ish +Ġridic ulously +ĠK art +Ġmitochond rial +Ġinterf ering +Build er +Ġover fl +Ġac ne +ĠM ud +ĠK err +f lex +ĠPost al +ĠBalt ic +47 7 +ĠPers ons +our age +H B +ĠM use +ĠImm ortal +ĠDri ving +Ġpet itions +Ġsubsc ript +Ġs orce +ĠProcess or +ut on +S ony +Ġph on +Ġr aced +ĠAnth rop +Ġday time +ĠEx ercise +Add ing +Ġeng ages +ĠQual comm +Ġmir acles +Ġmem es +ĠDr ink +ĠOri oles +Ġhair s +ĠPol ar +ath om +Ġsl ippery +ĠR emy +Ġcar amel +ĠY EAR +Ġal k +I gn +a ution +ĠMer lin +ĠC ran +Ġap ologies +Ġ4 10 +Ġout ing +ĠMem ories +app ointed +Ġcount ered +u ld +pos ing +Ġfire wall +ĠW ast +ĠW et +work ed +se ller +Ġrepe aled +ere o +ass uming +BL IC +m ite +ĠCEO s +ĠChap el +ellig ent +________________ ________ +D og +Ġw art +Ġsubsc riber +s ports +Ġbe gged +ĠM V +Ġsem if +eth ical +Ġpre ach +Ġrev ital +Ġpun itive +Ġshort cuts +Ġinstit uted +ĠWars aw +Ġabdom en +ĠK ING +Ġsuper intendent +Ġf ry +ĠGe o +T OR +Ġcontrad ictions +apt ic +Ġlandsc apes +b ugs +Ġcl ust +Ġvol ley +c ribed +Ġt andem +Ġrob es +WH AT +Ġpromot er +Ġel oqu +review ed +ĠD K +ĠPl ato +Ġf ps +T ank +ĠDer rick +Ġpriorit ize +as per +ĠHond uras +ĠCom pleted +ne c +Ġm og +n ir +ĠMay o +DE F +st all +in ness +ĠVolks wagen +Ġprec aution +ĠM ell +i ak +ist ries +Ġ24 8 +Ġoverl apping +Sen ate +ĠEnh ance +res y +rac ial +OR TS +ĠM ormons +Str ong +ĠCo ch +Mex ico +ĠMad uro +Ġj ars +Ġcan e +W ik +oll a +iff erence +Ġphysic ist +ĠMag gie +Ġ28 5 +Ġdep iction +ĠMcL aren +J u +Ġsl ows +Ġcommission ers +ĠWill ow +ĠExpl os +hov ah +Ġtechn ician +Ġhom icides +ĠFl av +ĠTr uman +Ġ100 00 +u ctor +Ġsh ader +News letter +45 7 +Ġre ver +Ġhard ened +Ġwhere abouts +Ġrede velop +Ġcar bs +Ġtra vers +Ġsqu irrel +Ġfoll ower +Ġs ings +50 8 +Ġrabb its +emon ium +Ġdocument ing +Ġmisunder stood +) ' +R ick +gg ies +Ġprem ie +Ġsk ating +Ġpass ports +Ġf ists +aged don +H aw +AC P +0 80 +ĠThough ts +ĠCarl son +Ġpriest hood +h ua +Ġdun geons +ĠLo ans +Ġant is +Ġfamiliar ity +ĠS abb +op al +ĠIn k +st rike +Ġc ram +Ġlegal ized +Ġcu isine +Ġfib re +Tra vel +ĠMon ument +OD Y +eth y +Ġinter state +ĠP UR +em porary +ĠArab ian +develop ed +Ġsadd le +Ġg ithub +ĠOff er +ĠIS P +ro let +ĠSUP ER +ĠDen is +Ġmultipl ier +Ġstir red +Interest ingly +Ġcustom ary +Ġbill ed +he x +Ġmultipl ied +Ġfl ipping +ĠCros by +Ġfundament als +ia e +ĠPlay ed +ĠAt om +am azon +ĠFl am +ee z +activ ated +Ġtables poon +Ġliberal ism +ĠPal in +ĠP atel +N um +ĠT AM +Ġs urn +ĠRel oaded +Ġco ined +" ], +ĠCl ash +ĠAg u +Ġprag matic +ĠActiv ate +Ġ8 02 +Ġtrail ers +Ġsil hou +Ġprob es +Ġcirc us +ĠB ain +ĠLind say +ĠAb bey +Del ivery +Ġconcess ion +Ġgast ro +ĠSpr ite +Ä Ł +and el +Ġg imm +Ġaut obi +ĠT urtle +Ġwonder fully +ĠHar am +ĠWorld wide +ĠHand le +Ġtheor ists +Ġsle ek +ĠZh u +ograph ically +EG A +ĠOwn ers +ath s +ĠAntar ctic +n atal +=" " +fl ags +`` `` +Ġs ul +K h +Ġpot assium +Ġlinem an +Ġcere al +ĠSe asons +Ġ20 22 +Ġmat hematic +Ġastron omers +prof essional +Ġf ares +cknow led +Ġch i +Ġyoung sters +Ġmistaken ly +Ġhem isphere +ĠDiv inity +r one +Ġ" , +r ings +Ġattract s +v ana +å ¹ +C AP +Ġplay list +Ġpor ch +ãģ £ +Ġincorpor ates +Ġso ak +Ġassert ing +ĠTerror ism +ĠP ablo +J a +ces ter +Ġfear ing +ĠPr ayer +Ġescal ated +G W +Ġro be +ĠBright on +ac ists +ĠSym phony +ĠDwar f +ĠPar ade +ĠLe go +Ġinex pl +Ġl ords +le af +RA G +l iber +Ġcig ars +ĠJe hovah +60 6 +WIND OWS +ĠLiber ia +eb us +He avy +Ġl ubric +ĠR W +angu ages +Ġnarrow ed +com puter +ĠE mber +Ġmurder ing +Ġdown stream +ĠT uls +ĠT ables +Top ic +ĠAcc uracy += / +l ost +ĠRe i +Ġprogress es +b ear +Ġestablish ments +Just in +ĠPe ach +ĠG omez +å ¿ +ĠTri angle +Id ent +ĠH ive +Res ources +Ġmix es +ĠAss uming +M u +Ġhyp oc +Ġs ane +ĠW an +id ious +Su ccess +Ġ io +Ang el +Ġdanger ously +ĠCreat ure +W ORK +: [ +ĠKat rina +List ener +M iller +ĠId lib +h ang +Ġcircum vent +h ref +Ġcel estial +ĠWe eks +ĠP ug +ĠDal ton +Ġsubpoen a +uk u +Ġpers isted +pe i +old ing +ĠDoc uments +ĠH ast +ĠC ENT +Ġprim er +Ġsyn onymous +Ġn ib +om bs +Ġnot ation +ĠD ish +ĠAt mosp +Ġforb id +ĠAN G +pat tern +l os +Ġproject iles +b rown +." , +ĠVen om +Ġfierce ly +ub lished +ĠU ran +ĠNic arag +4 10 +ĠC AL +OT OS +ĠMir acle +ĠEn chant +Ġguard ing +app end +Att ach +Ġlevel ed +Ġcond oms +ih ilation +64 9 +Ġnight mares +ĠTHE Y +ĠST ART +ĠK inn +Ġroomm ate +Ġhy giene +o pping +J ob +Ġl vl +ĠV ER +ĠKe eping +ab etic +Ġformat ting +eral a +Ġrev isions +Ġres urg +T el +ĠGood man +35 3 +p od +Ġind isp +ĠTrans lation +Ġg own +ĠM und +Ġc is +Ġby stand +col lect +ĠPun jab +act ively +ĠG amb +te ll +Ġimport ing +g encies +Ġloc om +ĠBr ill +H oly +ĠBer ger +Ġshow down +Ġrespond ers +IL Y +Ġt akedown +le ted +Ġmat tered +Ġpredict ive +Ġover lay +G PU +ĠV ick +Ġconvey ed +T ab +pe er +Sc an +Ġdefensive ly +v ae +Ġappro ving +Ġt iers +ĠV ia +quer ade +ĠSaud is +Ġdemol ished +ĠProp he +Ġmon o +Ġhospital ity +H AM +ĠAri el +M OD +ĠTor ah +Ġbl ah +ĠBel arus +erent ial +ĠT uc +Ġbank er +39 7 +Ġmosqu it +ĠScient ist +ĠMus ical +Ġh ust +Sh ift +Ġtor ment +Ġstand off +E duc +ĠF og +Ġampl ifier +Sh ape +Inst ance +ĠCrit ics +Ġda emon +H ouston +Ġmatt ress +ĠID F +Ġobsc ene +ĠA mer +hett i +Ġcomp iling +35 2 +vere tt +ĠRed uction +ist ration +ĠBl essed +ĠB achelor +3 16 +Ġpr ank +ĠVul can +dd ing +Ġm ourning +ĠQu int +ĠBl aster +test ing +Ġsed iment +>> > +ĠE ternity +ĠWH ERE +ĠM aze +Ġreact ing +ĠAl v +oms day +ĠC RA +Ġtransl ator +Ġbog us +at u +We bsite +oll s +Ġbapt ism +Ġs ibling +ĠAut umn +ve z +ãģ® é +gu ards +Ge org +assad ors +ĠFre ud +Ġcontin ents +ĠReg istry +Bern ie +ĸļ 士 +Ġtoler ant +ĠU W +Ġhor ribly +99 5 +ĠMID I +Ġimpat ient +oc ado +er i +ĠWor st +ĠNor ris +ĠTalk ing +Ġdef ends +ens able +Ġ20 21 +Ġanat omy +L ew +Ġdraw er +ĠCan berra +Ġpatri otic +é¾įå ĸļ士 +ĠAv g +AR M +Ġundis closed +Ġfare well +45 9 +b able +ĠAll ison +OL OG +Ġcon co +t ight +ĠAC PI +ĠM ines +l ich +ĠâĶ ľ +represent ed +200 000 +Ġenthusi ast +OT S +b il +ĠIng redients +Ġinvent or +ĠMy SQL +³³ Âł +ĠAB OUT +with in +Ġm k +B ul +ĠF ake +Ġdracon ian +W a +hel m +ĠTer ran +erv ille +Ġcommon place +SI ZE +Ġ" < +re place +ograph s +ĠSE LECT +inc ible +ĠMost ly +ĠShe ffield +ĠID E +ugg le +Ġcit ations +h urst +ĠUn ix +Ġunle ash +ĠP iper +ĠN ano +Ġsucc umb +Ġreluct ance +Ġ25 00 +ĠMer chant +Ġwire t +Ġcomb os +ĠBirth day +Ġchar coal +ĠU PS +ĠFair fax +Ġdrive way +ĠT ek +ĠP itch +ove re +Ġtechn icians +ĠAct ual +fl ation +ĠF iscal +ĠEm pty +an amo +Ġmag nesium +Ġsl ut +Ġgrow ers +Invest igators +( ): +ĠS atellite +ĠKe ynes +miss ive +l ane +Ġb orough +3 44 +ĠTE AM +ĠBet hesda +C V +h ower +ĠR AD +Ġch ant +ĠR iy +Ġcompos itions +Ġmild ly +Ġmedd ling +Ġag ility +ane ers +5 01 +Ġsyn th +ling er +29 1 +Ġex claimed +Part y +Ġcont amin +ĠMan or +ĠResp ond +Ġpra ising +Ġman ners +fle et +Sum mer +ĠLy nd +ĠDef initely +gr im +Ġbow ling +st ri +ç Ľ +y nt +Ġmand ates +D IV +Ġreconc ile +view s +ĠDam on +vet te +F lo +ĠGreat est +il on +ic ia +Ġportray al +Ġcush ion +50 4 +19 79 +oss al +App lic +sc ription +Ġmit igation +AT S +p ac +Ġer ased +Ġdefic iencies +ĠHolland e +ĠX u +Ġb red +Ġpregn ancies +f emin +Ġem ph +Ġpl anners +Ġout per +utter ing +Ġperpet rator +Ġm otto +ĠEll ison +ĠNE VER +Ġadmitted ly +AR I +ĠAzerbai jan +Ġmill isec +Ġcombust ion +ĠBott le +ĠL und +ĠP s +ĠD ress +Ġfabric ated +Ġbat tered +Ġs idel +ĠNot ting +Fore ign +ĠJer ome +0 20 +ĠAr bit +Ġkn ots +ĠR IGHT +M oving +ãģ Ļ +Ġsur geries +Ġcour thouse +Ġm astered +Ġhover ing +ĠBr an +ĠAl ison +Ġsaf est +m ilitary +Ġbull ied +Ġbar rage +Read er +ES E +ĠGe ographic +T ools +3 14 +ĠGe ek +ro th +gl ers +ĠF IN +Ï ģ +ĠA ston +al tern +48 8 +Ġveter in +G amer +Ġint el +ren ches +Sh ield +Ġam nesty +ĠB har +Ġp iled +Ġhonor able +ĠInst itutes +Ġso aked +Ġcom a +ĠE FF +34 1 +by tes +ĠG mail +le in +ĠCanad iens +m aterial +I l +Ġinstruct ors +ĠK Y +Ġconce ive +ub b +ĠP ossible +Ġeas ing +ĠChrist ina +Ġcar ic +ĠHD R +R OM +Ġsho vel +de lete +Ġp uff +ĠCh anging +Ġseam lessly +Att ribute +Ġacqu isitions +ak ery +ĠE F +Ġaut istic +ĠT akes +ĠPow der +ĠSt ir +5 10 +ĠBub ble +sett ings +ĠF owler +Ġmust ard +Ġmore over +Ġcopyright ed +ĠLED s +15 00 +æ ī +ĠH IS +en f +Ġcust od +ĠH uck +G i +Ġim g +An swer +C t +j ay +ĠInf rastructure +Ġfeder ally +L oc +Ġmicro bes +Ġover run +dd s +ot ent +adi ator +>>>> >>>> +Ġtorn ado +Ġadj ud +Ġintrig ued +Ġs i +ĠRevel ation +pro gress +Ġburgl ary +ĠSai yan +ĠK athy +Ġser pent +ĠAndre as +Ġcomp el +ess ler +ĠPl astic +ĠAd vent +ĠPos itive +ĠQ t +ĠHind us +reg istered +ular ity +Ġrighteous ness +Ġdemon ic +u itive +ĠB DS +ĠGre gg +c ia +ĠCrus ade +ĠSina i +W ARE ++ ( +Ġme ll +Ġder ail +y ards +A st +Ġnotice ably +ĠO ber +R am +Ġun noticed +Ġse q +av age +T s +Ġ6 40 +Ġconced e +Ġ] ) +F ill +Ġcapt ivity +ĠImprove ment +ĠCrus ader +ara oh +M AP +æ Ĺ +Ġstr ide +al ways +F ly +N it +Ġal gae +ĠCook ing +ĠDo ors +Mal ley +Ġpolic emen +ãģ į +Ġastron aut +access ible +49 5 +ĠR AW +cl iffe +udic rous +Ġdep ended +al ach +Ġvent ures +ra ke +Ġt its +ĠH ou +Ġcond om +ormon al +Ġind ent +Ġupload ing +Foot note +Import ant +Ġ27 1 +Ġmind ful +Ġcont ends +C ra +Ġcal ibr +ĠO ECD +plug in +F at +ĠIS S +ĠDynam ics +ans en +68 6 +' ), +Ġsp rite +Ġhand held +ĠH ipp +=~ =~ +Tr ust +Ġsem antics +ĠBund es +ĠRen o +ĠLiter ature +s ense +G ary +ĠA eg +ĠTr in +EE K +Ġcler ic +ĠSS H +Ġch rist +Ġinv ading +ib u +Ġen um +aur a +Ġal lege +ĠInc redible +B BC +Ġth ru +Ġsa iled +Ġem ulate +Ġin security +Ġc rou +Ġaccommod ations +Ġincompet ent +Ġsl ips +ĠEarth qu +s ama +IL LE +Ġi Phones +as aki +Ġby e +Ġar d +Ġext ras +Ġsl aughtered +Ġcrowd funding +res so +Ġfil ib +ĠER ROR +ĠT LS +e gg +ĠIt al +Ġen list +ĠCatal onia +ĠSc ots +Ġser geant +Ġdiss olve +N H +Ġstand ings +ri que +I Q +Ġbenef iciary +Ġaqu arium +You Tube +ĠPower Shell +Ġbright est +ĠWar rant +S old +Writ ing +Ġbegin nings +ĠRes erved +ĠLatin os +head ing +Ġ4 40 +Ġrooft op +AT ING +Ġ3 90 +VP N +G s +k ernel +turn ed +Ġprefer able +Ġturn overs +ĠH els +S a +ĠShin ji +ve h +ĠMOD ULE +V iol +Ġex iting +Ġj ab +ĠVan illa +Ġac ron +ĠG ap +ber n +A k +ĠMc Gu +Ġend lessly +ĠFar age +ĠNo el +V a +M K +Ġbr ute +ĠK ru +ĠES V +ĠOl ivia +âĢ ł +ĠK af +Ġtrust ing +Ġh ots +3 24 +Ġmal aria +Ġj son +Ġp ounding +ort ment +Count ry +Ġpostp oned +Ġunequ iv +? ), +ĠRo oney +udd ing +ĠLe ap +ur rence +sh apeshifter +ĠH AS +os ate +Ġca vern +Ġconserv atism +ĠB AD +Ġmile age +Ġarrest ing +V aults +Ġmix er +Dem ocratic +ĠB enson +Ġauth ored +8 000 +Ġpro active +ĠSpirit ual +t re +Ġincarcer ated +ĠS ort +Ġpe aked +Ġwield ing +re ciation +×Ļ × +P atch +ĠEm my +Ġex qu +tt o +ĠRat io +ĠP icks +ĠG ry +ph ant +Ġf ret +Ġeth n +Ġarch ived +% - +c ases +ĠBl aze +Ġim b +c v +y ss +im ony +Ġcount down +Ġaw akening +ĠTunis ia +ĠRe fer +ĠM J +Ġun natural +ĠCar negie +iz en +ĠN uggets +he ss +Ġev ils +64 7 +Ġintrodu ctory +l oving +ĠMcM ahon +Ġambig uity +L abel +ĠAlm ighty +Ġcolor ing +ĠCl aus +set ting +N ULL +ĠF avorite +ĠS IG +> ( +ĠSh iva +ĠMay er +Ġstorm ed +ĠCo verage +we apons +igh am +Ġun answered +Ġle ve +Ġc oy +c as +b ags +as ured +Se attle +ĠSant orum +ser ious +Ġcourage ous +ĠS oup +Ġconfisc ated +Ġ// / +Ġuncon ventional +Ġmom s +ĠRohing ya +ĠOrche stra +ĠPot ion +Ġdisc redit +ĠF IL +f ixed +ĠDe er +do i +ĠDim ension +Ġbureaucr ats +et een +Ġaction Group +oh m +Ġb umps +ĠUt ility +Ġsubmar ines +ren heit +re search +ĠShap iro +Ġsket ches +Ġde ceptive +ĠV il +es ame +ĠEss entially +Ġramp age +isk y +Ġmut tered +th ritis +Ġ23 6 +f et +b ars +Ġpup il +ĠTh ou +o S +s ong +Ġfract ured +Ġre vert +pict ure +Ġcrit erion +us her +Ġreperc ussions +ĠV intage +ĠSuper intendent +Offic ers +Ġflag ged +Ġbl ames +Ġin verse +ograp hers +Ġmakes hift +Ġdev oid +Ġfoss ils +ĠArist otle +ĠFund s +Ġde pleted +ĠFl u +ĠY uan +Ġw oes +Ġlip id +Ġsit u +requ isites +Ġfurn ish +ĠSam ar +Ġshame ful +Ġadverse ly +Ġad ept +Ġrem orse +Ġmurder ous +uck les +ĠE SL +Ġ3 14 +s ent +Ġred ef +ĠC ache +ĠP urs +ig ans +Ġ4 60 +Ġpres criptions +Ġf res +F uck +ocr ates +Tw enty +ĠWe ird +ĠT oggle +ĠC alled +itiz ens +Ġp oultry +Ġharvest ing +ãĤ¦ ãĤ¹ +Bott om +Ġcaution ed +t n +39 6 +ĠNik ki +Ġeval uations +Ġharass ing +Ġbind ings +ĠMon etary +Ġhit ters +Ġadvers ary +un ts +Ġset back +Ġenc rypt +ĠC ait +Ġl ows +eng es +ĠN orn +Ġbul bs +Ġbott led +ĠVoy ager +3 17 +Ġsp heres +p olitics +Ġsubt ract +Ġsens ations +Ġapp alling +Ġ3 16 +Ġenvironment ally +ĠST EM +Ġpub lishes +5 60 +Ġdilig ence +48 4 +Ġadv ises +Ġpet rol +Ġimag ining +Ġpatrol s +ĠInt eger +ĠAs hes +act us +ĠRad iant +ĠL T +it ability +ht aking +Set ting +Ġnu anced +ĠRe ef +ĠDevelop ers +N i +pie ces +99 0 +Lic ense +Ġlow ers +ĠOtt oman +3 27 +oo o +Ġqu itting +mark ets +Beh ind +Ġbas in +Ġdoc s +an ie +fl ash +ct l +Ġcivil ized +ĠFuk ushima +"] ," +ĠK S +ĠHonest ly +ar at +Ġconstruct s +ĠL ans +ĠD ire +ĠLI KE +ĠTrou ble +Ġwith holding +ĠOb livion +Ġsan ity +any a +Con st +Ġgro cer +ĠC elsius +Ġrecount ed +ĠW ife +B order +ate red +h appy +Ġspo iler +Ġlog ically +H all +Ġsucceed ing +Ġpoly morph +Ġax es +ĠShot gun +ĠS lim +ĠPrin ciples +ĠL eth +art a +Ġsc or +Sc reenshot +Ġrelax ation +#$ #$ +Ġdeter rent +idd y +Ġpower less +Ġles bians +Ġch ords +ĠEd ited +se lected +Ġseparat ists +000 2 +Ġair space +Ġturn around +Ġc unning +P ATH +P oly +Ġbomb ed +Ġt ion +x s +Ġwith hold +Ġw aged +ĠLiber ties +Fl ag +Ġcomfort ing +45 4 +ĠI ris +are rs +Ġr ag +Ġrel ocated +ĠGu arant +Ġstrateg ically +Ġgam ma +uber ty +ĠLock heed +g res +Ġgr illed +ĠLow e +st ats +ĠR ocks +Ġsens ing +Ġrent ing +ĠGe ological +ا Ø +ot rop +Ġse w +Ġimproper ly +48 6 +Ġâĸ ł +Ġstar ving +ĠB j +Disc ussion +3 28 +ĠCom bo +ĠFix es +N AT +Ġstri ving +th ora +Ġharvest ed +ĠP ing +Ġplay ful +Ġaven ues +Ġoccup ational +Ġw akes +ĠCou rier +Ġdrum mer +ĠBrow ser +ĠH outh +it u +Ġapp arel +p aste +Ġhun ted +ĠSecond ly +l ain +X Y +ĠP IN +ic ons +Ġcock tails +Ġs izable +Ġhurd les +est inal +ĠRecre ation +Ġe co +64 8 +ĠD ied +m int +Ġfinger prints +Ġdis pose +ĠBos nia +ts y +22 00 +Ġins pected +ĠF ou +Ġf uss +Ġamb ush +ĠR ak +Ġmanif ested +Pro secut +Ġsuff ice +ren ces +Ġcompens ated +ĠC yrus +Ġgen us +ĠWolver ine +ĠTrend s +Ġh ikes +ĠSe en +Ġen rol +C old +Ġpol itely +ĠSl av +ĠRu pert +Ġey ewitness +ĠAl to +Ġun comp +Ġposter ior +M ust +ĠHer z +Ġprogress ively +Ġ23 4 +Ġind ifference +ĠCunning ham +Ġacadem ia +Ġse wer +Ġast ounding +ĠA ES +r ather +Ġeld est +Ġclim bs +ĠAdd s +Ġout cry +Ġcont ag +ĠH ouses +Ġpe pt +ĠMel ania +interest ed +ĠU CH +ĠR oots +ĠHub bard +ĠT BD +ĠRoman ian +fil ename +St one +ĠIm pl +Ġchromos ome +C le +d x +Ġscram bled +ĠP t +Ġ24 2 +OP LE +Ġtremend ously +St reet +Ġcra ving +Ġbund led +ĠR G +p ipe +Ġinj uring +Ġarc ane +Part icip +ĠHero ic +st y +Ġto pping +ĠTemp est +rent ices +b h +Ġpar anoia +ĠUnic ode +Ġegreg ious +Ġ\ ' +ĠOsw ald +Ġgra vel +ĠSim psons +Ġbl and +ĠGuant anamo +Writ er +lin ers +ĠD ice +J C +Ġpar ity +Ġs ided +Ġ23 7 +ĠPyr rha +at ters +d k +F ine +comp an +Ġform ulated +ĠId ol +il ers +hem oth +ĠF av +Ġintr usion +Ġcar rots +ĠL ayer +ĠH acker +Ġ ---------------- +Ġmoder ation +é ģ +oc oc +Ġcharacter ize +ĠTe resa +Ġsocio economic +Ġper k +ĠParticip ation +tr aining +ĠPaul o +ph ys +Ġtrust worthy +Ġembod ied +ĠMer ch +c urrency +ĠPrior ity +Ġte asing +Ġabsor bing +Ġunf inished +ĠCompar ison +Ġdis ple +writ ers +Ġprofess ions +ĠPengu in +Ġang rily +ĠL INK +68 8 +ĠCor respond +Ġprev ailed +Ġcart el +l p +as ms +ĠRed emption +ĠIslam ists +effect s +d ose +ĠL atter +ĠHal ifax +Ġv as +ĠTop ics +ĠN amed +advert ising +zz a +IC ES +Ġret arded +ach able +ĠPupp et +ĠItem Level +Ġret ract +Ġident ifiable +A aron +ĠB uster +s ol +hel le +as semb +H ope +r anged +B a +ĠP urch +é Ģ +ĠSir i +Ġarri vals +Ġ19 12 +Ġshort ened +Ġ3 12 +Ġdiscrep ancy +ĠTem perature +ĠWal ton +Ġkind erg +p olit +Ġrem ix +Ġconnect ors +ãĥĺ ãĥ© +ĠKazakh stan +dom inated +Ġsu gars +im ble +ĠPan ic +ĠDem and +ĠCol ony +on en +ĠM ER +7 75 +ur ia +aza ar +ĠDeg ree +P ri +Ġsun shine +Ġ25 1 +Ġpsychedel ic +Ġdigit ally +ĠBra un +Ġsh immer +Ġsh ave +ĠTel esc +ĠAst ral +ĠVenezuel an +ĠO G +Ġc rawling +Int eg +ĠFe ather +Ġunfold ing +Ġappropri ation +Ġè£ı è +ĠMob ility +ĠN ey +- . +b ilt +L IN +ĠT ube +ĠCon versely +Ġkey boards +ĠC ao +Ġover th +Ġla ure +>> \ +ĠV iper +ach a +Off set +ĠR aleigh +ĠJ ae +J ordan +j p +Ġtotal itarian +Connect or +Ġobserv es +ĠSpart an +ĠIm mediately +ĠSc al +C ool +Ġt aps +Ġro ar +P ast +Ġch ars +ĠB ender +ĠShe ldon +Ġpain ter +Ġbe acon +ĠCreat ures +Ġdownt urn +Ġh inder +ĠAnd romeda +à Ľ +cc oli +ĠF itness +et rical +Ġutil izes +Ġsen ate +Ġen semble +Ġche ers +T W +Ġaff luent +k il +ry lic +ord ering +Com puter +Ġgru esome +ost ics +ĠUb isoft +ĠKel ley +Ġw rench +Ġbourgeois ie +IB LE +ĠPrest on +w orn +ar ist +reat ing +Ġst ained +ar ine +Ġsl ime +EN N +Ġche sts +Ġground water +ann ot +ĠTr ay +ĠLoc ke +ĠC TR +Ġd udes +ĠEx ternal +ĠDec oder +Ġpar amed +ĠMed line +80 9 +ĠD inner +rup al +g z +ĠG um +ĠDem o +j ee +Ġd h +ber man +arch s +Ġen qu +ĠEp stein +Ġdevast ation +Ġfriends hips +ĠAr d +Ġ23 1 +ĠRub in +ĠDist ance +Ġsp urred +Ġd ossier +Ġover looking +\\\\\\\\ \\\\\\\\ +Fore st +ĠCom es +\ ", +ĠIran ians +Ġf ixtures +L aughs +Ġcur ry +ĠKing ston +Ġsqu ash +Ġcat alogue +Ġabnormal ities +Ġdigest ive +.... ..... +Ġsubord inate +og ly +Ġ24 9 +M iddle +Ġmass ac +Ġburg ers +Ġdown stairs +Ġ19 31 +39 4 +ĠV G +Ġl asers +ĠS ikh +ĠAlex a +der ived +Ġcycl ist +ãģ® éŃĶ +onel iness +!!!! !!!! +Ġbuff s +leg ate +Ġrap ing +Ġrecomm ending +ro red +Ġmult icultural +un ique +Ġbusiness men +Ġune asy +ĠM AP +Ġdisp ersed +cipl ine +J ess +ĠK erala +å § +Ġabst raction +Sur v +U h +Ġprin ters +ij a +ow der +Ġanalog ous +ĠA SP +af er +Ġunfold ed +Ġlevel ing +Ġbre ached +ĠH earing +Ġn at +Ġtransl ating +crit ical +Ġant agonist +ĠYes terday +Ġfuzz y +w ash +m ere +Ġbe wild +ĠM ae +V irgin +ph rase +Ġsign aled +ĠH IGH +Ġprot ester +Ġgar ner +unk nown +Ġk ay +Ġabduct ed +Ġst alking +am n +Ġdes erving +ĠR iv +ĠJ orge +Ġscratch ing +ĠS aving +ip ing +Ġte ase +Ġmission ary +ĠMor row +T IME +P resent +Ġchem otherapy +tern ess +ĠH omes +ĠP urdue +Ġst aunch +ĠWhit ney +ĠTH ERE +Î ¼ +iat us +ĠErn est +ĠDe ploy +Ġcove ted +F ML +ĠDial ogue +Ġex ited +f ruit +Ġner d +":" "," +Ġv ivo +ru ly +4 60 +ĠAm en +rehens ible +Ġâ ĺ +D IR +Ġad herence +Ġche w +ĠCo ke +ĠSerge i +dig ital +ĠNe ck +g ently +enth al +/ ) +Ġwe ary +Ġgu ise +ĠConc ord +ĠOn ion +at cher +Ġb inge +ĠDirect ive +Ġman ned +ans k +Ġill usions +Ġbillion aires +38 3 +oly n +odynam ic +ĠWhe at +ĠA lic +Ġcol oured +ĠN AFTA +ab o +Ġmac ros +ind ependent +s weet +Ġsp ac +ĠK abul +Ġ Ä +em e +Ġdict ated +Ġsh outs += { +Ġr ipping +ĠSh ay +ĠCr icket +direct ed +Ġanalys ed +ĠWAR RANT +ag ons +ĠBlaz ers +Ġche ered +Ġar ithmetic +ĠTan z +37 3 +ĠFl ags +Ġ29 5 +Ġw itches +ĠIn cluded +ĠG ained +ĠBl ades +G am +ĠSam antha +ĠAtl antis +ĠPr att +Ġspo iled +ĠI B +ĠRam irez +Pro bably +re ro +ĠN g +ĠWar lock +t p +Ġover he +Ġadministr ations +Ġt int +Ġreg iment +Ġpist ols +Ġblank ets +Ġep ist +Ġbowl s +Ġhydra ulic +Ġde an +Ġj ung +Ġasc end +70 5 +ĠSant iago +à ® +Ġun avoid +ĠSh aman +re b +Ġstem ming +99 8 +ĠM G +st icks +esthes ia +ER O +Ġmor bid +ĠGr ill +ĠP oe +any l +Ġdele ting +ĠSurve illance +Ġdirect ives +Ġiter ations +ĠR ox +ĠMil ky +F ather +Ġpat ented +44 7 +Ġprec ursor +Ġm aiden +ĠP hen +ĠVe gan +ĠPat ent +K elly +Redd itor +Ġn ods +Ġvent ilation +ĠSchwar z +Ġw izards +Ġomin ous +ĠHe ads +ĠB G +Ġl umber +ĠSp iel +Ġis Enabled +Ġancest ral +ĠSh ips +Ġwrest ler +ph i +Ġy uan +ĠRebell ion +Ġice berg +Ġmag ically +Ġdivers ion +ar ro +yth m +ĠR iders +ĠRob bie +ĠK ara +ĠMain tenance +ĠHer b +Ġhar ms +p acked +ĠFe instein +Ġmarry ing +Ġbl ending +ĠR ates +Ġ18 80 +Ġwr ink +ĠUn ch +ĠTor ch +desc ribed +Ġhuman oid +ilit ating +ĠCon v +ĠFe ld +IGH TS +Ġwhistlebl ower +ort mund +ets y +arre tt +ĠMon o +ĠI ke +ĠC NBC +ĠW AY +ĠMD MA +ĠIndividual s +Ġsupplement al +Ġpower house +ĠSt ru +F ocus +aph ael +ĠCol leg +att i +Z A +Ġp erenn +ĠSign ature +ĠRod ney +Ġcub es +idd led +ĠD ante +ĠIN V +iling ual +ĠC th +Ġso fa +Ġintimid ate +ĠR oe +ĠDi plom +ĠCount ries +ays on +Ġextrad ition +Ġdis abling +ĠCard iff +Ġmemor andum +ĠTr ace +Ġ?? ? +se ctor +ĠRou hani +ĠY ates +ĠFree ze +Ġbl adder +M otor +ĠProm ise +ant asy +Ġforesee able +ĠC ologne +cont ainer +ĠTre es +ĠG ors +ĠSin clair +Ġbar ring +key e +Ġsl ashed +ĠStat istical +é ĩ +Ġâĸ º +All ows +Ġhum ility +Ġdr illed +ĠF urn +44 3 +Ġse wage +Ġhome page +Ġcour tyard +Ġv ile +Ġsubsid iaries +aj o +direct ory +Ġam mon +V ers +charg es +Ġ} } +ĠCh ains +Ġ24 6 +n ob +Ġper cept +Ġg rit +Ġfisher men +ĠIraq is +ĠDIS TR +ĠF ULL +ĠEval uation +g raph +at ial +Ġcooper ating +Ġmel an +Ġenlight ened +Ġal i +t ailed +Ġsal ute +Ġweak est +ĠBull dogs +U A +ĠAll oy +Ġsem en +oc ene +ĠWilliam son +s pr +, âĢĶ +ĠG F +itt ens +Be at +ĠJ unk +iph ate +ĠFarm ers +ĠBit coins +ig ers +d h +ĠL oyal +p ayer +Ġentert ained +Ġpenn ed +Ġcoup on +Que ue +Ġweaken ing +c arry +Ġunderest imate +Ġshoot out +Ġcharism atic +ĠProced ure +Ġprud ent +in ances +Ġric hes +Ġcort ical +Ġstr ides +Ġd rib +ĠOil ers +5 40 +ĠPer form +ĠBang kok +Ġe uth +S ER +Ġsimpl istic +t ops +camp aign +Q uality +Ġimpover ished +ĠEisen hower +Ġaug ment +ĠH arden +Ġinterven ed +Ġlist ens +ĠK ok +Ġs age +Ġrub bish +ĠD ed +Ġm ull +pe lling +Ġvide ot +Produ ction +D J +m iah +Ġadapt ations +Ġmed ically +Ġboard ed +Ġarrog ance +Ġscra pped +Ġopp ress +FORM ATION +Ġj unction +4 15 +EE EE +S kill +Ġsub du +ĠSug gest +ĠP ett +Ġle tt +ĠMan ip +ĠC af +ĠCooper ation +T her +Ġreg ained +¶ æ +ref lect +Ġth ugs +ĠShel by +Ġdict ates +ĠWe iner +ĠH ale +Ġbatt leground +s child +Ġcond ol +h unt +osit ories +Ġacc uses +Fil ename +Ġsh ri +Ġmotiv ate +Ġreflect ions +N ull +ĠL obby +¥ µ +ĠS ATA +ĠBack up +Ñ ĥ +n in +ĠCor rection +Ġju icy +ut ra +ĠP ric +Ġrest raining +ĠAir bnb +ĠAr rest +Ġappropri ations +Ġsl opes +Ġmans laughter +Ġwork ings +ĠH uss +ĠF rey +Le ave +ĠHarm ony +ĠF eder +Ġ4 30 +Ġt rench +Ġglad ly +Ġbull pen +ĠG au +b ones +Ġgro ove +Ġpre text +ã ħĭ +Ġtransm itter +ĠComp onent +Ġunder age +ĠEm pires +T ile +Ġo y +ĠMar vin +ĠC AS +Ġbl oss +Ġrepl icated +ĠMar iners +Marc us +ĠBl ocks +Ġliber ated +Ġbutter fly +Fe el +Ġfer mentation +Ġyou tube +Ġoff end +ĠTer m +res ist +Ġcess ation +Ġinsurg ency +Ġb ir +ĠRa ise +59 5 +Ġhypothes es +50 2 +Ġpl aque +ocr at +Ġjack ets +ĠHuff Post +am ong +Ġconf er +48 7 +ĠL illy +Ġadapt ing +ĠF ay +Ġsh oved +ve c +Ġref ine +Ġg on +Ġgun men +z ai +ĠShut tle +ĠI zan +Ġ19 13 +Ġple thora +· · +Ġ5 10 +Ġp uberty +Ġ24 1 +ĠWe alth +ĠAl ma +ĠM EM +ĠAd ults +C as +pr ison +R ace +Ġwater proof +Ġathlet icism +Ġcapital ize +ĠJu ice +Ġillum inated +ĠP ascal +Ġirrit ation +ĠWitness es +ad le +ĠAst ro +Ġf ax +ĠEl vis +Prim ary +ĠL ich +ĠEl ves +Ġres iding +Ġst umble +3 19 +ĠP KK +Ġadvers aries +D OS +ĠR itual +Ġsm ear +Ġar son +ident al +Ġsc ant +Ġmon archy +Ġhal ftime +Ġresid ue +Ġind ign +ĠSh aun +ĠEl m +aur i +A ff +W ATCH +ĠLy on +hel ps +36 1 +Ġlobby ist +Ġdimin ishing +Ġout breaks +Ġgo ats +f avorite +ĠN ah +son ian +ĠBo oster +Ġsand box +ĠF are +ĠMalt a +Ġatt Rot +ĠM OR +ld e +Ġnavig ating +T ouch +Ġunt rue +ĠDis aster +Ġl udicrous +Pass word +ĠJ FK +blog spot +4 16 +ĠUN DER +ern al +Ġdelay ing +T OP +Ġimpl ants +ĠAV G +ĠH uge +att r +Ġjournal istic +ĠPe yton +ĠI A +R ap +go al +ĠProgram me +Ġsm ashing +w ives +print ln +ĠPl ague +in us +EE P +Ġcru iser +ĠPar ish +umin ium +Ġoccup ants +ĠJ ihad +m op +Ġp int +Ġhe ct +ĠMe cca +direct or +ĠFund ing +ĠM ixed +Ġst ag +T ier +Ġg ust +Ġbright ly +ors i +Ġup hill +R D +Ġles ions +ĠBund y +liv ious +Ġbi ologist +ĠFac ulty +ĠAuthor ization +Ġ24 4 +All ow +ï ¸ +ĠGi ul +Ġpert inent +ot aur +es se +ĠRo of +Ġunman ned +35 1 +ĠSh ak +ĠO rient +Ġend anger +D ir +Ġrepl en +ed ient +Ġtail or +Ġgad gets +Ġaud ible +âĺ Ĩ +N ice +Ġbomb ard +ĠR ape +Ġdef iance +ĠTW O +ĠFilip ino +Ġunaff ected +erv atives +Ġso ared +ĠBol ton +Ġcomprom ising +ĠBrew ers +R AL +ĠA HL +icy cle +Ġv ampires +Ġdi pped +oy er +ĠX III +Ġsidew ays +ĠW aste +ĠD iss +ĠâĶľ âĶĢâĶĢ +$ . +Ġhabit ats +ĠBe ef +tr uth +tr ained +spl it +R us +And y +ĠB ram +RE P +p id +è£ ħ +ĠMut ant +An im +ĠMar ina +Ġfut ile +hig hest +f requency +Ġepile psy +Ġcop ing +Ġconc ise +Ġtr acing +ĠS UN +pan el +ĠSoph ie +ĠCrow ley +ĠAd olf +ĠShoot er +Ġsh aky +ĠI G +ĠL ies +ĠBar ber +p kg +Ġupt ake +Ġpred atory +UL TS +/ ** +Ġintox icated +ĠWest brook +od der +he ment +Ġbas eman +AP D +st orage +ĠFif ty +ed itor +G EN +UT ION +ir ting +Ġse wing +r ift +Ġag ony +ĠS ands +Ġ25 4 +C ash +Ġl odge +Ġp unt +N atural +ĠIde as +Ġerrone ous +ĠSens or +ĠHann ity +Ġ19 21 +Ġm ould +ĠG on +kay a +Ġanonym ously +ĠK EY +Ġsim ulator +W inter +Ġstream ed +50 7 +? ", +Ġte ased +Ġco efficient +Ġwart ime +ĠTH R +' '. +ĠBank ing +mp ire +Ġf andom +Ġl ia +G a +Ġdown hill +Ġinterpre ting +Ind ividual +N orm +Ġjealous y +bit coin +Ġple asures +ĠToy s +ĠChev rolet +ĠAd visor +IZ E +Ġrecept ions +70 6 +C ro +Ġ26 2 +Ġcit rus +ir u +Review er +ject ed +U ES +an z +19 81 +ĠWork er +Ġcompl ied +ores cent +contin ental +T on +ĠPr ism +ĠShe ep +Ġ28 8 +n ox +ĠV og +O rd +Ġreal ms +te k +Ġirrig ation +Ġbicy cles +Ġelectron ically +p oly +t all +() ); +Ġaest hetics +ĠInteg rated +Expl ore +Ġd unk +47 6 +p ain +ĠJac ques +ĠD mit +Fram es +Ġreun ited +Ġhum id +D ro +P olitical +Ġyouth ful +Ġent ails +Ġmosqu ito +36 3 +spe cies +Ġcoord inating +ĠMay hem +ĠMagn us +M ount +Impro ved +ĠST ATE +ATT LE +Ġflow ed +Ġtack led +Ġfashion ed +Ġre organ +iv ari +f inger +Ġreluct antly +et ting +ĠV and +you ng +ĠGar land +Ġpresum ption +Ġamen ities +ĠPle asant +on ential +ĠO xy +Ġmor als +ĠY ah +Read y +Sim on +En h +D emon +Ġcl ich +Mon itor +ĠD U +Ġwel comes +Ġstand out +Ġdread ful +Ġban anas +Ġball oons +h ooting +bas ic +Ġsuff ix +Ġd uly +can o +Ch ain +at os +Ġgeop olitical +Ġ( & +ĠGem ini +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +Ġacqu itted +L uck +prot ect +10 24 +Ġsc arcity +Ġmind fulness +ec ided +D N +pr ime +ĠPres idents +ĠVID EO +Ġ( âĪĴ +add ock +N OR +ĠP ru +p un +ĠL OL +)) )) +ĠL iqu +ĠS AS +Ġsty ling +Ġpunish ments +Ġnum b +Ġasc ertain +ĠRock ies +f lu +Th umbnail +Ġperpet rated +ĠSem i +Ġdis arm +ĠOld er +ĠEx ception +Ġexponent ially +ĠCommun ities +Ġabol ish +ĠPart ner +pt oms +Ġ7 77 +ĠFo ley +ĠC ases +Ġgre ase +ĠReb irth +G round +Ġ; ) +ĠDoct rine +ik ini +Y e +ĠBl ossom +Ġpers ists +b ill +Ġinf usion +Ġbud dies +9 11 +ĠPat ient +Ġdem os +Ġacquaint ance +ĠP aw +at ari +Ġx ml +Ġfasc ination +ĠSer ve +Ï Ĥ +br anded +Ġa z +Return s +Ġover shadow +Ġro am +Ġspeed y +n umbered +hel ial +Ġdisc iple +Ġass urances +g iven +pect ing +ĠN atalie +çĶ ° +Ġmosquit oes +rote in +Ġnumer ic +Ġindepend ents +Ġtrans itional +Ġreaction ary +ĠMech dragon +do ctor +Ġshort est +Ġsequ ential +ĠB ac +ĠAccount s +ãģ Į +ach y +ract ive +ĠReg iment +Ġbreat htaking +ffic iency +ĠB ates +Ġ3 11 +Ġward robe +ft s +ĠBer k +Sim ply +ĠRivers ide +iver ing +ident ial +lu cent +Ġen riched +ĠCon ver +ĠG iving +ãĥ Ļ +Ġlegal ize +ĠF TC +Ġfre aking +M ix +Ġter restrial +es ian +ci ents +W ing +LO AD +Ġled ge +ĠViol ent +ĠMet all +Ġ30 8 +Ġs outheastern +hett o +M eat +Ġslow down +Ġret reated +Jere my +end as +**** * +er ic +Ġre ins +opp able +ĠHuman ity +ear ances +rig an +C amera +Ġwa ivers +s oc +Ġalter ation +trans form +ĠC emetery +50 6 +Ġindef inite +Ġstim ulating +y g +60 3 +ĠS op +Ġdescript ive +Ph ase +ĠEd mund +Ġpneum onia +vent us +A mb +Ġlabor atories +ĠEx clusive +ug ar +W ere +Ġmalf unction +Ġhomosexual s +Ġ---- --- +un i +Ġturb ines +ĠEqu ity +D u +Ġmind ed +ĠR H +ĠBlack hawks +Ġfe ats +Ġ17 00 +re pl +36 2 +lad en +Ġindisp ensable +ly ss +tt i +Ġre el +Ġdiver ted +Ġlik eness +Ġsubscript ions +Ġfing ert +Ġfil thy +dest ruct +d raft +ĠBernard ino +l aunch +Ġper plex +ĠS UM +car b +Ġswe ater +ĠVent ure +ĠJ ag +ĠCele b +ĠV oters +Ġstead fast +Ġathlet ics +ĠHans on +ĠDr ac +Tr acker +Ġcomm end +ĠPres idency +ĠD ID +in formed +Ġweb page +P retty +Ġforce fully +ãĥĥ ãĤ¯ +Ġrel ocation +Ġsat ire +â ī +ĠSunder land +æ Ħ +V oice +???? ???? +Ġinform ant +Ġbow el +ĠUn iform +Ġ ..." +Ġpur ge +Ġpic nic +ĠU mb +ĠU PDATE +ĠSapp hire +ĠSt all +le arn +Ġobject ively +Ġob liter +Ġlooph ole +Ġjour neys +Ġo mission +Pro s +ĠSid ney +pl oma +Ġspray ed +Ġg uru +Ġtra itor +Ġtim et +Ġsn apping +ĠSe vent +urn al +ĠUk ip +Ġb owed +por al +l iberal +R os +Quest ions +i OS +Ġsummar ize +ST AT +Ġ18 50 +ap est +Ġl ender +ĠVari able +br inging +ĠL ORD +, ) +Ġcollaps es +x iety +ĠN ed +Y D +ĠSch a +Ġantib ody +Ġdis band +y re +ill usion +Ġro ver +s hed +ĠHiro sh +cc i +Ġcal am +ĠMort on +P interest +Ġ19 28 +ĠE uras +ord es +Ġf ences +ĠIn ventory +ĠVal encia +ĠU d +ĠT iff +Ġsqu e +Ġqu otation +Ġtroubles ome +er ker +QU EST +ĠKing doms +s outh +Ġle vy +Pr ince +ĠSt ing +Ġnick named +Ġapp e +Ġphot ographic +Ġcorp us +re ference +ĠT rog +U nt +) =( +ĠLat via +Ġactiv ating +Ġlicense e +Ġdispar ities +ĠNews letter +ãĥĥ ãĥĪ +Ġfree ing +ĠJe ep +ĠPer ception +ins k +Ġsil icone +ĠHay den +Le an +ĠSuz uki +ibr arian +66 8 +Ġsp or +Ġcorrel ations +ag hetti +Ġtu ber +ĠIP CC +il us +ĠV u +Ġwealth iest +ĠCarb uncle +an za +Ġfool ed +ĠZ ur +Ġd addy +ran o +il ian +Ġknock out +f man +requ ired +ĠWik ileaks +ĠD uffy +ON T +Ġins ol +ĠObject s +Ġb ou +ĠNord ic +ĠIns ert +sc an +Ġd ancers +Ġid iots +major ity +ĠNev ille +ĠFree BSD +Ġt art +pan ic +69 0 +Ġcoc oa +Ġsam pled +Ġlook up +Ind ust +Ġinject ions +gen re +Ġa u +Ġroad way +Ġgen itals +K ind +ĠEx aminer +ĠY az +F resh +Ġpar alysis +ĠAl uminum +Ġre ap +ok é +Ġsl oppy +ĠTun nel +pos ium +ner y +en ic +Ġher bal +ĠOut er +ĠBuild er +Ġinc ur +Ġide ologies +Ġback ups +cons uming +ĠDet ect +de ck +ĠKN OW +ĠG ret +ĠM IC +Ġtough ness +ĠEx hibit +Ġh ive +L es +ĠSCH OOL +ĠAt ari +ald e +ĠN ull +and estine +m ouse +Ġbrig ade +48 9 +Ġrev ol +ĠLaw son +ĠW ah +op oly +eb ted +ĠS aunders +Ġ3 13 +ĠW inc +Ġtab oo +ĠHel met +Ġw edge +ch ip +ĠT ina +b g +Ġinf uri +r n +Ġanomal ies +ĠSy nc +ĠEx am +ĠComm it +ĠDi ary +ĠALS O +ĠDe bor +omed ical +Ġcomprehens ion +6 55 +Ġempower ing +Ġ ire +Ġju ices +ĠE TH +ĠBox ing +=" / +Ġfacilit ated +p oke +ĠPars ons +ĠMod er +tra vel +Ġcivil izations +Ġliber tarians +Ġrun e +ĠCl arks +at hed +Ġcampaign ers +ĠDis patch +ĠFah renheit +ĠCap com +-------- -- +Ġl ace +Ġdr aining +Ġl iner +ĠArt ificial +é n +t ask +] ). +ĠGM O +ĠOper ator +ord inary +ĠInf luence +ĠU ps +Ġpot ency +uss en +osp ons +ĠSw im +ĠDead line +Un ity +Ġcul inary +Ġenlight enment +Ġwe arer +Ġmin ed +Ġp ly +Ġinc est +ĠDVD s +W alk +B TC +Tr ade +Ġdev al +ib and +ĠOvers ight +Palest inian +Ġd art +Ġm ul +L R +Ġrem ovable +ĠReal ms +ì Ŀ +Ġmisc ar +ĠV ulkan +68 5 +è re +ĠS ap +Ġmer ging +ĠCar ly +che ster +Ġbr isk +Ġlux urious +ĠGener ator +Ġbit terness +Ġed ible +Ġ24 3 +T G +Ġrect angle +With No +bel ow +J enn +Ġdark est +Ġh itch +Ġdos age +Ġsc aven +ĠK eller +ĠIllust rated +Certain ly +ĠMaver icks +Marg inal +Ġdiarr hea +Ġenorm ously +Ġ9 99 +sh r +qu art +Ġadam ant +ĠM ew +Ġren ovation +Ġcerv ical +ĠPercent age +en ers +ĠKim ber +Ġflo ats +Ġde x +ĠW itcher +ĠSwan sea +d m +Ġsal ty +y ellow +Ġca pe +ĠDr ain +ĠPaul a +ĠTol edo +les i +Mag azine +ĠW ick +ĠM n +ĠA ck +ĠR iding +AS ON +Ġhom ophobic +AR P +Ġwand ered +C PU +ood oo +ĠP ipe +Ġtight ening +ĠBut t +3 18 +Ġdesert ed +S ession +Ġfacilit ating +J ump +Ġemer gencies +OW ER +Ġexhaust ive +ĠAF TER +Ġheart beat +ĠLab el +ack y +ĠCert ified +ilt ration +Z e +ĠU tt +Ġ13 00 +Ġpres ume +ĠDis p +Ġsur ged +Ġdoll s +Col umb +Ġchim pan +ĠR azor +Ġt icks +Ġcouncill or +Ġpilgr image +ĠReb els +ĠQ C +ĠA uction +x ia +ik k +b red +Ġinsert ion +Ġco arse +d B +SE E +ĠZ ap +ĠF oo +Ġcontem por +ĠQuarter ly +ot ions +ĠAl chemist +ĠT rey +ĠDu o +S weet +80 4 +ĠGi ov +Ġfun n +N in +h off +Ġram ifications +Ġ19 22 +ĠExper ts +az es +Ġgar ments +ar ial +ĠN ab +Ġ25 7 +ĠV ed +Ġhum orous +ĠPom pe +Ġn ylon +Ġlur king +ĠSerge y +ĠMatt is +Ġmisogyn y +ĠComp onents +ĠWatch ing +ĠF olk +ract ical +B ush +Ġt aped +Ġgroup ing +Ġbe ads +Ġ20 48 +Ġcon du +quer que +Read ing +Ġgriev ances +Ult ra +Ġend point +H ig +ĠSt atic +ĠScar borough +L ua +ĠMess i +a qu +ĠPsy Net +ĠR udd +Ġa venue +v p +J er +Ġsh ady +ĠRes ist +ĠArt emis +Ġcare less +Ġbro kers +Ġtemper ament +Ġ5 20 +T ags +ĠTurn ing +Ġut tered +Ġp edd +Ġimpro vised +Ġ: ( +Ġtab l +Ġpl ains +16 00 +press ure +ĠEss ence +marg in +friend s +ĠRest oration +Ġpoll ut +ĠPok er +ĠAugust ine +ĠC IS +ĠSE AL +or ama +Ġth wart +se ek +Ġp agan + º +cp u +Ġg arn +Ġass ortment +ĠI LCS +t ower +Recomm ended +Ġun born +ĠRandom Redditor +ĠRandomRedditor WithNo +Ġparaly zed +Ġeru ption +Ġinter sect +ĠSt oke +ĠS co +B ind +å ¾ +ĠP NG +ĠNeg ative +ĠNO AA +Le on +Ġall oy +ĠL ama +ĠD iversity +5 75 +Ġunderest imated +ĠSc or +Ġm ural +Ġb usted +so on +l if +Ġnone x +Ġall ergy +ĠUnder world +ĠR ays +ĠBl asio +Ġh rs +ĠD ir +Ġ3 27 +by ter +Ġrepl acements +Ġactiv ates +ri ved +M H +Ġp ans +ĠH I +Ġlong itudinal +Ġnu isance +al er +Ġsw ell +ĠS igned +s ci +ĠIs les +ĠA GA +Ġdef iant +Ġson ic +oc on +K C +ĠA im +t ie +ah ah +Ġm L +D X +Ġb isc +ĠBill board +ĠSY STEM +NE Y +ga ard +Ġdist ressed +former ly +Al an +Ġche fs +Ġopt ics +ĠC omet +ĠAM C +Ġredes igned +irm ation +Ġsight ings +38 2 +3 11 +ĠW B +Ġcont raction +ĠT OTAL +D ual +Ġstart led +Ġunderstand ably +Ġsung lasses +ETH OD +Ġd ocker +Ġsurf ing +ĠH EL +ĠSl ack +ton es +Ġsh alt +Vis ual +49 8 +Dep artment +c ussion +Ġunrest ricted +Ġt ad +Ġre name +employ ed +Ġeduc ating +Ġgrin ned +bed room +ĠActiv ities +ĠV elvet +ĠSW AT +Ġsh uffle +ig or +Ġsatur ation +F inding +c ream +ic ter +Ġv odka +tr acking +te c +Ġfore ground +iest a +Ġve hement +ĠEC B +ĠT ie +E y +Ġt urtles +ĠRail road +ĠKat z +ĠFram es +Ġmen ace +ĠFell owship +ĠEss ential +ugg ish +Ġdri p +ch witz +ĠKy oto +s b +ĠN ina +Param eter +Ġal arms +ĠCl aud +Ġpione ering +Ġchief ly +ĠSc ream +Col lection +Ġthank fully +ĠRonald o +åŃ IJ +st rip +ĠDisney land +com mercial +See ing +S oul +Ġevac uate +Ġc iv +ĠAs he +Ġdiv ides +ĠD agger +rehens ive +Ġber ries +ĠD F +Ġs ushi +Ġplur ality +W I +Ġdisadvant aged +Ġbatt alion +ob iles +45 1 +Ġcl ing +Ġunden iable +ĠL ounge +Ġha unt +p he +Ġquant ify +Ġdiff ered +Ġ[* ] +ĠV iz +c um +sl ave +Ġvide og +Ġqu ar +Ġbund les +ĠAl onso +t ackle +Ġneur onal +Ġlandsl ide +conf irmed +ĠDep th +Ġrenew ables +B ear +ĠMaced onia +Ġjer seys +Ġb unk +ĠSp awn +ĠControl s +ĠBuch anan +Ġrobot ics +Ġemphas izing +ĠTut orial +h yp +ist on +Ġmonument al +æ ° +ĠCar ry +Ġt bsp +en ance +H ill +art hed +Ġro tten +De an +Ġtw isting +Ġgood will +Ġimm ersion +L iving +Ġbr ushes +ĠC GI +ĠAt k +tr aditional +Ġph antom +ĠSt amina +Ġexpans ions +ĠMar in +Ġembark ed +ĠE g +int estinal +ĠPE OPLE +ĠBo oth +ĠApp alach +Ġreleg ated +V T +M IT +Ġmust er +Ġwithdraw ing +Ġmicrosc ope +ĠG athering +ĠC rescent +ĠArgent ine +ĠDec re +ĠDomin ic +Ġbud s +ant age +ĠI on +Ġwid ened +ONS ORED +ĠGl oves +iann opoulos +raz en +fe el +Ġrepay ment +Ġhind sight +ĠRE ALLY +ĠPist ol +ĠBra h +Ġwat ts +Ġsurv ives +Ġfl urry +iss y +Al ert +ĠUrug uay +Ph oenix +S low +ĠG rave +ĠF ir +Ġmanage able +Ġtar iff +ĠU DP +ĠPist ons +ĠNiger ian +Ġstrike outs +Ġcos metics +whel ming +f ab +c ape +pro xy +Ġre think +Ġover coming +sim ple +Ġw oo +Ġdistract ing +ĠSt anton +ĠTuls a +ĠD ock +65 9 +Ġdisc ord +ĠEm acs +ĠV es +ĠR OB +Ġreass uring +Ġcons ortium +Muslim s +3 21 +Ġprompt s +se i +ĠH itch +imp osed +ĠF ool +Ġindisc rim +wr ong +bu querque +D avis +! ] +Ġtim eless +ĠNE ED +Ġpestic ide +Ġrally ing +ĠCal der +Ġå ¤ +Ġx p +ĠUn le +ĠEx port +lu aj +B uff +) [ +Ġsq or +S audi +Ġis tg +Ġindul ge +pro c +Ġdisg usted +Ġcomp ounded +Ġn em +Ġschool ing +ĠC ure +process ing +S ol +Ġpro verb +it ized +ĠAlv arez +Ġscar f +Ġrect angular +re ve +Ġh ormonal +ĠSt ress +itiz en +Ġ4 25 +girl s +ĠNo ir +ĠR app +Ġmar ches +ch urch +ĠUs es +Ġ40 5 +ĠBer m +Ġord inances +ĠJud gment +Charg es +ĠZ in +Ġdust y +Ġstraw berries +Ġper ce +ĠTh ur +ĠDebor ah +net flix +ĠLam bert +Ġam used +ĠGu ang +Y OU +R GB +ĠC CTV +Ġf iat +r ang +Ġf ederation +ĠM ant +ĠB ust +ĠM are +respect ive +ĠM igration +ĠB IT +59 0 +Ġpatriot ism +Ġout lining +reg ion +ĠJos é +Ġbl asting +ĠEz ra +B s +Ġundermin es +ĠSm ooth +Ġcl ashed +rad io +Ġtransition ing +ĠBucc aneers +ĠOw l +Ġplug s +Ġh iatus +ĠPin ball +Ġm ig +ĠNut r +ĠWolf e +Ġinteg ers +Ġor bits +ĠEd win +ĠDirect X +b ite +Ġbl azing +v r +Ed ge +ĠP ID +ex it +ĠCom ed +ĠPath finder +ĠGu id +ĠSign s +ĠZ er +ĠAg enda +Ġreimburse ment +M esh +i Phone +ĠMar cos +ĠS ites +h ate +en burg +Ġs ockets +p end +Bat man +v ir +ĠSH OW +Ġprovision al +con n +ĠDeath s +AT IVE +Pro file +sy m +J A +Ġnin ja +inst alled +id ates +eb ra +ĠOm aha +Ġse izing +ĠBe asts +Ġsal ts +M ission +Gener ally +ĠTr ilogy +he on +leg ates +Ġd ime +Ġf aire +par able +G raph +Ġtotal ing +Ġdiagram s +ĠYan uk +ple t +ĠMe h +Ġmyth ical +ĠStep hens +aut ical +ochem istry +Ġkil ograms +Ġel bows +anc ock +ĠB CE +ĠPr ague +Ġimpro v +ĠDev in +Ġ" \ +par alle +Ġsuprem acists +ĠB illion +Ġreg imen +inn acle +Ġrequ isite +ang an +ĠBur lington +ain ment +ĠObject ive +oms ky +G V +Ġun ilateral +Ġt c +Ġh ires +ment al +Ġinvol untary +Ġtrans pl +ĠASC II + ¨ +Ev ents +Ġdoub ted +ĠKa plan +ĠCour age +ig on +ĠMan aging +ĠT art +Ġfalse hood +ĠV iolet +Ġair s +Ġfertil izer +Brit ain +Ġaqu atic +ou f +W ords +ĠHart ford +Ġeven ings +ĠV engeance +qu ite +G all +ĠP ret +Ġp df +ĠL M +ĠSo chi +ĠInter cept +9 20 +Ġprofit ability +ĠId le +ĠMac Donald +ĠEst ablishment +um sy +Ġgather ings +ĠN aj +Charl ie +Ġas cent +ĠProt ector +Ġal gebra +Ġbi os +for ums +EL S +Introdu ced +Ġ3 35 +Ġastron omy +Cont ribut +ĠPol ic +Pl atform +Ġcontain ment +w rap +Ġcoron ary +ĠJ elly +man ager +Ġheart breaking +c air +ĠChe ro +c gi +Med ical +ĠAccount ability +! !" +oph ile +Ġpsych otic +ĠRest rict +Ġequ itable +iss ues +Ġ19 05 +ĠN ek +c ised +ĠTr acking +Ġo zone +Ġcook er +ros is +Ġre open +Ġinf inity +ĠPharm aceutical +ens ional +Att empt +ĠR ory +Mar co +Ġawa its +H OW +t reated +Ġbol st +Ġreve red +Ġp ods +opp ers +00 10 +Ġampl itude +ric an +SP ONSORED +Ġtrou sers +Ġhal ves +ĠK aine +ĠCut ler +ĠA UTH +Ġsplend id +Ġprevent ive +ĠDud ley +if acts +umin ati +ĠY in +Ġad mon +ĠV ag +Ġin verted +Ġhast ily +ĠH ague +L yn +Ġled ger +Ġastron omical +get ting +Ġcirc a +ĠC ic +ĠTenn is +Lim ited +Ġd ru +ĠBY U +Ġtrave llers +Ġp ane +ĠInt ro +Ġpatient ly +Ġa iding +Ġlo os +ĠT ough +Ġ29 3 +Ġconsum es +Source File +Ġ"" " +Ġbond ing +Ġtil ted +Ġmenstru al +ĠCel estial +UL AR +Plug in +Ġrisk ing +N az +ĠRiy adh +Ġacc redited +Ġsk irm +é Ľ +Ġexam iner +Ġmess ing +Ġnear ing +ĠC hern +ĠBeck ham +Ġsw apped +Ġgo ose +K ay +Ġlo fty +ĠWal let +Ġ[ ' +Ġap ocalypse +Ġb amboo +ĠSP ACE +ĠEl ena +Ġ30 6 +ac ons +Ġtight ened +Ġadolesc ence +Ġrain y +Ġvandal ism +ĠNew town +Ġcon ject +c akes +Ġche ated +Ġmoder ators +par ams +E FF +Ġdece it +ĠST L +ĠTanz ania +ĠR I +Ġ19 23 +ĠEx ile +the l +Ġthe olog +Ġquir ky +ĠIr vine +Ġneed y +or is +U m +K a +Ġmail box +3 22 +Ġb os +ĠPet ra +K ING +Ġenlarg ed +O ften +Ġbad ass +Ġ3 43 +ĠPl aces +ĠC AD +Ġpr istine +Ġinterven ing +d irection +Ġl az +ĠD SM +Ġproject ing +ĠF unk +ag og +pay ment +n ov +Ġch atter +AR B +Ġexam inations +ĠHouse hold +ĠG us +F ord +4 14 +B oss +Ġmy stic +Ġle aps +ĠB av +ul z +b udget +Foot ball +Ġsubsid ized +Ġfirst hand +Ġcoinc ide +oc ular +Con n +ĠColl abor +Ġfool s +am ura +ah ar +r ists +Ġsw ollen +Ġexp ended +ĠP au +s up +Ġsp ar +Ġkey note +s uff +Ġunequ al +Ġprogress ing +str ings +ĠGamer gate +Dis ney +ĠEle ven +om nia +Ġscript ed +Ġear ners +bro ther +ĠEn abled +æ ³ +Ġlar vae +ĠL OC +m ess +Wil son +ĠTem plate +success fully +Ġparam ount +Ġcamoufl age +Ġbind s +ĠQu iet +ĠSh utterstock +r ush +Ġmasc ot +fort une +ĠCol t +ĠBe yon +hab i +Ġha irc +Ġ26 7 +ĠDe us +Ġtw itch +Ġconcent rating +Ġn ipples +c ible +Ġg ir +N Z +M ath +n ih +Requ ired +Ġp onder +ĠS AN +Ġwedd ings +Ġl oneliness +N ES +ĠMah jong +69 5 +add le +ĠGar ner +ĠC OUR +Br idge +Ġsp ree +ĠCald well +Ġbri bery +Ġ���� ���� +plug ins +Ġr acket +Ġchamp agne +vers ible +V ote +Ġmod ifiers +May or +6 80 +Ġassemb lies +ĠS ultan +ĠN ing +ĠLad ies +Ġsulf ur +Ġor bs +Ġ---- - +____ ___ +ĠJournal ism +Ġes ports +Ġl ush +Ġh ue +Ġspect ral +H onest +ãĥ ı +Ġbus hes +Ġrein forcement +Ġre opened +ĠWhe els +ĠM org +rie ving +Ġaux iliary +Ġj Query +ĠB AT +tes que +Ġver tex +p ure +f rey +ãĤ º +d os +Ġty ph +Ġc ull +Ġe q +Ġdec on +Ġtoss ing +Ġdispar ate +ĠBr igham +print f +led ged +Ġsu nd +Ġco zy +Ġhepat itis +per forming +Ġav al +ĠG G +f uture +Ġpet ertodd +ĠKos ovo +Ġmagn ets +Al ready +ĠEd ison +ĠCe res +ĠRA ID +Ġbrill iance +57 6 +Ġder ives +Ġhypert ension +ĠÎ Ķ +Ġlamb da +Ġfl air +Ġmission aries +Ġrap es +ĠSt arter +ĠMon ths +Ġdef y +Ġseism ic +ĠR aphael +Ġeuro zone +65 6 +z sche +Ġscr atched +Ġb ows +ĠLenn on +ĠGa ia +Ġdri pping +f acts +A le +Ġfrog s +ĠBre ast +ogene ity +ĠProsecut or +Ġampl ified +ĠHod g +ĠF n +Th ousands +ĠNI H +ĠMonitor ing +FT WARE +ĠPri ebus +ĠG rowing +hun ter +Ġdiagn ose +ĠM ald +ĠL R +Ġcrown ed +Ġburst ing +Ġdiss olution +j avascript +Ġuseful ness +ĠExec ution +: ( +ĠIv ory +a ah +Ġpersecut ed +viol ence +ist as +ĠCr ate +Ġimpuls es +ĠSp ani +ed es +Hand le +ĠZ erg +think able +Last ly +Ġspont aneously +Ġinconven ient +Ġdismiss ing +Ġpl otted +Ġeight y +Ġ7 37 +r ish +ĠThor nton +ath am +Ġsit com +V en +Rec ipe +t el +l und +Ġcle ars +ĠSas uke +Ġ25 8 +Ġopt ing +Ġen raged +est hetic +ĠA e +uch s +Pre p +Fl ow +Ġrun off +ĠE ating +ĠG iles +ĠAct ing +res ources +ib aba +Ġr pm +Ġske wed +ĠBl anc +ĠS akuya +Ġhot ter +Ġ19 24 +op ian +ck o +Ġcr umbling +Ġcapt ains +ĠAppropri ations +le aders +dro pping +an uts +Ġrevers ing +ĠP ose +ĠS ek +Sc ot +ĠIde a +c ise +ĠSloven ia +Ġ3 17 +Do ctor +Ġcro cod +ald i +Se a +ĠFar rell +Ġmerc enaries +ĠR NC +ĠGu ess +Ġp acing +M achine +Streamer Bot +ĠChar ity +Ġ29 8 +Ġcann ons +ĠTob y +TPP StreamerBot +ĠPass ion +cf g +Th om +Ġbad ges +ĠBern stein +. âĢĵ +ĠP OP +ĠCon j +Ġinitial ization +Ġbiod iversity +D ub +Ġfeud al +Ġdisclaim er +Ġc row +Ġign ition +ar f +S HA +Ġk Hz +h azard +ĠArt ists +oe uv +67 9 +ĠRud y +N ine +ĠRam adan +å ½ +itt o +Ġadren aline +C ert +Ġsmell ed +Ġimp unity +Ġag endas +ĠRe born +ĠCon cent +ĠSe ems +Ġo mega +ĠDust in +Ġback er +ĠSau ce +ĠBoy le +W IN +Ġsp ins +Ġpa uses +u pt +Ġshred ded +Ġstra pped +ĠCor ruption +Ġscr atches +Ġn i +Ġatt ire +ĠS AF +Factory Reloaded +ĠI PS +Ġ( % +Ġsem inar +f ocus +c ivil +Ġ18 60 +int osh +Ġcontin ual +Ġabbre vi +ĠS ok +oc obo +X M +Ġfr antic +Ġunavoid able +Ġar tery +Ġannot ations +b ath +Cl imate +Ġd ors +ĠSl ide +co ord +ĠRel oad +ĠL DL +ĠLove craft +Ġunim agin +Ġresemb led +Ġbarr acks +n p +Ġsurrog ate +Ġcategor ized +ãĤ © +Ġvacc inated +Ġdrain age +Ġind ist +ĠWhats App +Ġ18 70 +oler ance +inv oke +am orph +Ġrecon nect +Ġem anc +Ġblind ness +Ġ12 80 +intern et +c ollar +Ġalt ru +Ġab yss +ĠT RI +65 7 +Ġinf used +HE AD +Ġforest ry +ĠWood y +ĠC i +w i +s am +78 4 +hol iday +Ġmog ul +ĠF ees +ĠD EN +In ternal +ur bed +f usc +at om +ĠIll usion +Ġpoll ed +Ġfl ap +Ġco ax +L GBT +An aly +ĠSect ions +ĠCalif orn +em n +Ġh ither +ĠN IGHT +Ġn ailed +ĠPip eline +39 1 +o of +ĠPr imal +vere nd +Ġsl ashing +Ġret ri +avi our +Ġdepart ing +g il +IS C +Ġmid way +Ġultras ound +Ġbeh aving +ĠT ara +class es +V irtual +ĠColon ial +Ġstri pping +Ġorchestr ated +ĠGra ves +45 2 +ĠIron ically +ĠWrit ers +Ġl ends +ĠMan z +Ġra ven +Ġoxid ative +Ġ26 6 +EL F +act ually +asc ar +D raft +Ġfavour able +Ġhumili ating +Ġf idelity +ĠH of +ĠX uan +49 6 +Ġlay ered +at is +79 0 +Ġpay check +it on +K ar +ĠVM ware +ĠFar mer +Ġserv ic +gl omer +Ġsl ump +ĠFab ric +ĠD OC +est ing +Ġreass ure +Ġph yl +v olt +it ory +R ules +Ġoxid ation +Ġpri zed +Ġmist ress +ĠDj ango +WAR N +å ij +Ġenc ode +ĠFeed back +Ġstupid ity +I an +ĠYugoslav ia +× ¨ +ac l +UT E +19 77 +Ġqual ifies +Ġpuls es +pret ty +Ġfro ze +Ġs s +Iter ator +Ġur gently +Ġm ailed +ĠCh am +Ġsust aining +Ġbas il +Ġpupp ies +il ant +ĠP LEASE +l ap +ace ous +F ear +ĠMaster y +aut omatic +ĠT AG +Ġant im +ag les +47 3 +fram es +Ġwh ispers +ĠWho ever +Ġbra very +ĠUK IP +ract ions +"" " +Ġt ame +Ġpart ed +every thing +CON T +Ġind ebted +Ġadd r +re k +IR ED +Ġem inent +cl inton +Ġo usted +Ġreview er +Ġmelt down +Ġre arr +ĠY ao +the real +aby te +Ġst umbling +Ġbat ches +Ġ25 9 +Ġcontrace ptive +Ġprost itute +ens is +De cl +ĠSt rikes +M ilitary +ĠO ath +v acc +pp ings +05 2 +Ġpart Name +amp ing +Rep orts +K I +CH R +Ġsubt ly +sw ers +Bl ake +us ual +Ġcontest ants +Ġcart ridges +ĠGRE AT +Ġbl ush +ĠâĢ º +47 2 +Ġreason ed +ãĥ ¤ +paralle led +Ġd yn +ag ate +Ġnight ly +å Ĩ +55 6 +Ġsem antic +ĠAdv oc +Ġ !! +Ġdisag rees +ĠB W +V eh +Ġharm ing +Ġembr aces +Ġstri ves +Ġin land +ĠK ard +Ġhe ats +ĠGin ny +ut an +ern aut +yl ene +ĠE lev +J D +Ġh ars +ĠStar r +Ġsk ysc +Ġcollabor ators +Us ually +Ġrev olutions +ĠSTAT S +Ġdism antle +Ġconfident ly +Ġkin etic +Al i +Ġpercent ile +Ġextract ing +ill ian +est ead +Ġphysic ists +ĠMarsh al +Ġfell owship +Ġd ashed +ĠU R +ĠSi oux +ĠComp act +am ide +P ython +ĠLe igh +ĠPharm ac +ist rates +her ical +Ġf ue +ĠE min +Ġ( { +ĠNeighbor hood +Ġdisrupt ing +ĠD up +Ġg land +ĠSe v +ĠMar ian +arg on +ĠD und +Ġ< !-- +Ġstr and +Ġstadium s +z os +Ġpsych osis +ĠR ack +Ġbrilliant ly +ï¸ ı +Ġsubmer ged +ĠInst it +ĠCh ow +Ġc ages +ĠH ats +ĠU rs +Ġdil uted +us at +ien ne +ĠMembers hip +ĠBur k +Ġ ie +Ġarche type +D rug +ult on +ĠSp ock +ĠMcK ay +ĠDep end +F eatured +S oc +19 78 +ĠB ere +Ġrelent lessly +Ġcripp ling +Ġar thritis +çĶ Ł +ĠTrop ical +ĠBul g +ĠCher yl +Ġadm irable +Ġsub title +Over ride +Ġorig inating +ĠC CP +Ġsw ore +ĠSo le +ĠDis orders +3 29 +Ġprocess ion +Ġref urb +Ġimm ersed +requ ently +Ġskept ics +Ġcer amic +m itter +en stein +b elt +ĠT IT +b idden +Ġf ir +m ist +> ] +Ġwe ave +ĠParad ox +Ġentr usted +ĠBarcl ays +Ġnovel ist +og ie +80 6 +Ġnin ety +Ġdisag reements +@@@@ @@@@ +ĠAus chwitz +c ars +ĠL ET +t ub +arant ine +P OS +Ġback story +Ġcheer ful +ĠR ag +ek a +bi ased +Ġinexper ienced +ak ra +ĠW itt +t an +Ġrap ist +Ġplate au +ch al +ĠInqu is +exp ression +Ġc ipher +Ġsh aving +add en +re ly +( \ +ism a +ĠReg ulatory +CH AR +ily n +N VIDIA +G U +Ġmur m +la us +Christ opher +Ġcontract ual +ĠPro xy +ĠJa ime +ĠMethod ist +Ġstew ards +st a +per ia +Ġphys iology +Ġbump ed +Ġf ructose +Austral ian +ĠMet allic +ĠMas querade +ar b +Ġprom ul +Ġdown fall +Ġbut cher +Ġb our +ĠIN FORMATION +ĠB is +pect s +ad ena +Ġcontempl ating +ar oo +cent ered +ĠPe aks +Us ed +Ġmod em +Ġg enders +Ġ8 000 +37 1 +Ġm aternity +ĠR az +Ġrock ing +Ġhandgun s +ĠD ACA +Aut om +ĠN ile +Ġtum ult +ĠBenef it +ĠAppro ach +works hop +ĠLe aving +G er +inst ead +Ġvibr ations +Ġrep ositories +49 7 +ĠA unt +ĠJ ub +ĠExp edition +Al pha +Ġs ans +Ġoverd ue +Ġoverc rowd +Ġlegisl atures +Ġp aternal +ĠLeon ardo +Ġexp ressive +Ġdistract ions +Ġsil enced +tr ust +Ġb iking +Ġ5 60 +Ġpropri et +Ġimp osition +Ġcon glomer +Ġ= ================================================================ +ĠTe aching +ĠY ose +int ensive +T own +Ġtroll ing +ĠGr ac +ĠAS US +Y o +Ġspecial s +ĠNep h +ĠGod zilla +Dat abase +ĠHe gel +Ġ27 2 +19 76 +ĠGl oria +Ġdis emb +ĠInvestig ations +ĠB ane +ag ements +St range +Ġtre asury +ĠPl ays +Ġundes irable +Ġwid ening +Ġverb ally +Ġinf ancy +Ġcut ter +f ml +Ġ21 00 +prot otype +f ine +Ġdec riminal +Ġdysfunction al +Ġbes ie +ĠErn st +z eb +Ġnort heastern +Ġa ust +por ate +ĠMar lins +Ġsegreg ated +ew orld +ĠMa her +Ġtra verse +Ġmon astery +ur gy +G ear +s and +Com pl +ĠE MP +Ġpl ent +ĠMer cer +Ġ27 6 +TA BLE +Config uration +H undreds +Ġpr ic +Ġcollabor ating +ĠPar amount +ĠCumm ings +Ġ( < +Ġrecord er +Ġfl ats +Ġ4 16 +wh ose +Font Size +ĠOr bit +Y R +Ġwr ists +Ġb akery +) } +ĠB ounty +ĠLanc aster +Ġend ings +acc ording +ĠSal am +e asy +75 5 +ĠBur r +ĠBarn ett +onom ous +Un ion +Ġpreced ence +ĠScholars hip +ĠU X +Ġroll out +Ġbo on +al m +ĠCan ter +æ µ +Ġround ing +Ġcl ad +Ġv ap +ĠF eatured +is ations +Ġ5 40 +pol ice +Ġunsett ling +Ġdr ifting +ĠLum ia +ĠObama Care +ĠF avor +Hy per +ĠRoth schild +ĠMil iband +an aly +ĠJul iet +H u +Ġrec alling +a head +69 6 +Ġunf avorable +Ġd ances +O x +Ġleg ality +Ġ40 3 +rom ancer +Ġinqu ire +ĠM oves +\ "> +ĠVari ant +ĠMess iah +ĠL CS +ĠBah á +75 6 +Ġeyeb row +Ġ ¥ +ĠMc F +ĠFort y +M as +Ġpan icked +Ġtransform ations +q q +Ġrev olves +ring e +ĠA i +ax e +Ġon ward +ĠC FR +ĠB are +log in +Ġliqu ids +Ġde comp +second ary +il an +ĠCon vert +ami ya +Ġprosecut ing +Ġâī ¡ +ĠYork ers +ĠByr ne +sl ow +aw ei +J ean +Ġ26 9 +ĠSky dragon +Ġ é +ĠNicarag ua +ĠHuck abee +ĠHigh ly +Ġamph ib +ĠPast or +ĠL ets +Ġbl urred +Ġvisc eral +ĠC BO +Ġcollabor ated +z ig +Leg al +Ġapart heid +Ġbr id +Ġpres et +ĠD ET +ĠAM A +× Ķ +arch ing +auc uses +build er +Ġpo etic +Ġem ulator +ĠMole cular +Ġhon oring +ise um +Ġtract or +ĠCl uster +ĠCal m +ared evil +Ġsidew alks +Ġviol in +Ġgeneral ized +ĠAle c +Ġemb argo +Ġfast ball +ĠHT TPS +ĠL ack +ĠCh ill +ri ver +C hel +ĠSw arm +ĠLev ine +ro ying +L aunch +Ġkick er +Ġadd itive +ĠDe als +W idget +cont aining +Ġescal ate +ĠOP EN +Ġtwe aked +Ġst ash +Ġsp arks +ĠEs sex +ĠE cc +Ġconv ict +Ġblog ging +I ER +ĠH L +Ġmurd erers +75 9 +ĠH ib +Ġde pl +ĠJ ord +S ac +Ġdis sect +ĠHow e +os her +Ġcustom izable +ĠFran z +Ġat ro +Ä ĩ +Ġ000 4 +Ġout post +R oss +Ġglyph osate +ĠHast ings +ĠBE FORE +Ġsh ove +o pped +ĠSc ala +Ġam ulet +an ian +Ġexacerb ated +Ġe ater +47 1 +UM E +Ġpul p +izont al +ĠZ am +ĠAT I +imm une +aby tes +Ġunnecess arily +ĠC AT +ĠAx is +Ġvisual ize +à ī +ĠRad ical +f m +Doc uments +ĠFor rest +Ġcontext ual +ĠSy mbol +Ġtent ative +ĠDO ES +ĠGood s +Ġintermitt ent +} : +medi ated +Ġridic ule +Ġathe ism +Ġpath ogens +ĠM um +Ġre introdu +Ġ30 7 +i HUD +Ġflash light +Ġsw earing +Ġp engu +B u +Ġrot ated +ĠCr ane +Ġ() ); +Ġfashion able +Ġendors ing +46 3 +) [ +Ġingest ion +Ġcook s +Ġ9 50 +ot omy +ĠIm am +Ġk a +Ġte aser +ĠGhost s +ĠãĤ µ +19 69 +Ï ĥ +ub by +Ġconver ter +zan ne +end e +ĠPre par +ĠNic kel +ĠChim era +h im +ĠTyr ann +ĠSabb ath +ĠNich ols +Ġra pt +ih ar +Ġshe lling +Ġillum inate +Ġdent ist +ut or +ĠInteg ration +Ġwh ims +ĠLiter ary +Be aut +Ġp archment +ag ara +Br and +Ġder og +â̦ ) +ĠNor se +Ġunw itting +Ġc uc +Ġborder line +Ġupset ting +Ġrec ourse +Ġd raped +ĠRad ar +Ġcold er +ĠPep si +im inary +], [ +65 8 +V i +ĠF rem +ĠP es +Ġveter inary +ĠT ED +ĠEp idem +n ova +k id +Ġdev out +o ct +j ad +M oh +ĠP AY +Ġge ometric +Ġ3 23 +Ġcircum ference +ich ick +19 75 +ĠY uri +ĠSh all +ĠH over +un in +S pr +Ġg raft +ĠHapp iness +Ġdisadvant ages +att acks +Ġhub s +ĠStar Craft +é ĸ +Ġgall eries +ĠKor ra +Ġgrocer ies +ĠGors uch +Ġrap ists +Ġfun gi +ĠTyph oon +V ector +ĠEm press +b attle +4 68 +Ġparas ite +ĠBom ber +S G +ex ist +ĠP f +Ġun se +Ġsurge ons +B irth +ĠUn sure +ĠPrint ed +ĠBehavior al +ĠA ster +Pak istan +Ġun ethical +Ġs v +ĠIo T +Ġlay outs +P ain +Ġconst ants +ĠL W +ĠB ake +Ġtow els +Ġdeterior ation +ĠBol ivia +Ġblind ed +ĠW arden +ĠMist ress +Ġon stage +Ġcl ans +ĠB EST +19 60 +Ġant ique +Ġrhet orical +ĠPer cy +ĠRw anda +, . +B ruce +Ġtra umat +ĠParliament ary +Ġfoot note +id ia +ĠLear ned +se eking +gen ic +Ġdim ensional +H ide +èĢ ħ +Ġintrig ue +in se +Ġle ases +Ġapp rentices +w ashing +Ġ19 26 +V ILLE +Ġsw oop +s cl +Ġbed rooms +on ics +ĠCr unch +comp atible +Ġincap ac +ĠYemen i +ash tra +z hou +d anger +Ġmanifest ations +ĠDem ons +AA F +Secret ary +ACT ED +L OD +Ġam y +ra per +eth nic +4 17 +Ġpos itives +Ġ27 3 +ĠRefuge es +Ġus b +ĠV ald +odd y +ĠMahm oud +As ia +Ġskull s +ĠEx odus +ĠComp et +ĠL IC +ĠM ansion +ĠA me +Ġconsolid ate +storm s +ont ent +99 6 +Ġcl en +Ġm ummy +fl at +75 8 +ĠV OL +oter ic +n en +ĠMin ute +S ov +Ġfin er +R h +ly cer +Ġreinforce ments +ĠJohann es +ĠGall agher +Ġgym n +S uddenly +Ġext ortion +k r +i ator +T a +Ġhippocamp us +N PR +ĠComput ing +Ġsquare ly +Ġmod elling +ĠFor ums +ĠL isp +ĠKrish na +Ġ3 24 +Ġr ushes +Ġens ued +Ġcre eping +on te +n ai +il ater +ĠHorn ets +Ġob livious +IN ST +55 9 +Ġjeopard y +Ġdistingu ishing +j ured +Ġbeg s +sim ilar +ph ot +5 30 +ĠPark way +Ġs inks +ĠHearth stone +ib ur +ĠBat on +Av oid +Ġd ancer +Ġmag istrate +ary n +Ġdisturb ances +ĠRom ero +Ġpar aph +Ġmis chief +âĸ ĵ +ĠSh aria +Ġur inary +r oute +iv as +f itted +Ġeject ed +ĠAl buquerque +Ġ4 70 +Ġirrit ated +ĠZ ip +ĠB iol +à į +Ġden ounce +Ġbin aries +ĠVer se +Ġopp os +ĠKend rick +ĠG PL +Ġsp ew +ĠEl ijah +ĠE as +Ġdr ifted +so far +Ġannoy ance +ĠB ET +47 4 +ĠSt rongh +it ates +ĠCogn itive +oph one +ĠIdent ification +ocr ine +connect ion +Ġbox er +ĠAS D +ĠAre as +Y ang +t ch +ull ah +Ġdece ive +Comb at +ep isode +cre te +W itness +Ġcondol ences +ht ar +Ġhe als +Ġbuck ets +ĠLA W +B lu +Ġsl ab +ĠOR DER +oc l +att on +ĠSteven son +ĠG inger +ĠFriend ly +ĠVander bilt +sp irit +ig l +ĠReg arding +ĠPR OG +Ġse aling +start ing +Ġcard inal +ĠV ec +ĠBe ir +Ġmillisec onds +we ak +per se +Ġster ile +ĠCont emporary +ĠPh ant +ĠCl o +Ġout p +Ġex iled +Ġ27 7 +Ġself ie +Ġman ic +Ġn ano +ter ms +Alex ander +Ġres olves +Ġmillenn ia +Ġexpl odes +Ġconst ellation +Ġadul tery +m otion +D OC +Ġbroad casters +Ġkinderg arten +ĠMay weather +ĠE co +ich o +Ġ28 7 +l aun +Ġm ute +Ġdisc reet +Ġpres chool +Ġpre empt +De lete +ĠFre ed +P i +H K +Ġblock er +ĠC umber +Ġw rought +d ating +Ġins urer +Ġquot as +Ġpre ached +Ġev iction +ĠReg ina +ĠP ens +Ġsevent een +ĠN ass +D ick +Ġfold s +Ġd otted +ĠA ad +Un iversal +Ġp izz +ĠG uru +Ġso ils +Ġno vice +ĠNe ander +Ġst ool +Ġdeton ated +ĠPik achu +ĠMass ive +IV ER +ĠAb del +Ġsubdu ed +Ġtall est +Ġprec arious +Ġa y +r ification +ĠOb j +c ale +Ġun question +cul osis +ad as +igr ated +D ays +Ġque ens +ĠGaz ette +ĠCol our +ĠBow man +ĠJ J +ï ve +Ġdomin ates +Stud ent +Ġm u +Ġback log +ĠElect ro +Tr uth +48 3 +Ġcond ensed +r ules +ĠCons piracy +Ġacron ym +hand led +ĠMat te +j ri +ĠImp ossible +l ude +cre ation +Ġwar med +ĠSl ave +Ġmis led +Ġfer ment +ĠK ah +ink i +ke leton +cy l +ĠKar in +Hun ter +Reg ister +ĠSur rey +Ġst ares +ĠW idth +ĠN ay +ĠSk i +Ġblack list +uck et +Ġexp ulsion +im et +Ġret weet +vant age +Fe ature +Ġtro opers +Ġhom ers +9 69 +Ġconting ency +ĠW TC +ĠBrew er +fore ign +W are +S olar +Ġund ue +RE C +ulner able +path ic +ĠBo ise +Ġ3 22 +Ġarous ed +ĠY ing +ä¸ į +uel ess +Ġp as +Ġmor p +Ġfl oral +Ex press +ud ging +k B +ĠGr anted +Ø ¯ +ĠMich a +ĠGoth ic +ĠSPEC IAL +ĠRic ardo +F ran +Ġadminister ing +6 20 +por a +Ġ ® +Ġcomprom ises +Ġb itten +Ac cept +Th irty +Ð ² +Ġmater ially +ĠTer r +ig matic +ch ains +Ġdo ve +stad t +Mar vel +FA ULT +Ġwind shield +Ġ3 36 +ad ier +Ġsw apping +Ġflaw less +ĠPred ator +ĠMiche le +Ġprop ulsion +ĠPsych ic +Ġassign ing +Ġfabric ation +Ġbar ley +l ust +Ġtow ering +Ġalter cation +ĠBent ley +Sp here +Ġtun a +ĠClass es +Fre edom +un er +L ady +v oice +Ġcool est +or r +Ġpal p +$ { +Ġhyster ia +ĠMet atron +p ants +Ġspawn ing +Exper ts +ĠInvest ors +ĠAn archy +Ġshr unk +ĠVict im +Ġ28 9 +Ġec stasy +ĠB inding +58 5 +ĠMel ody +57 8 +ot ally +ĠE tsy +lig a +Ġapplaud ed +Ġswe ating +Ġredist ributed +Ġpop corn +Ġsem inal +f ur +ĠNeuro science +R and +ĠO st +ĠMadd en +ĠIncre asing +ĠDaw kins +ĠSub way +Ġar sen +cons erv +B UR +Ġsp iked +ĠLy ft +ĠImper ium +ĠDrop box +Ġfav oured +Ġencomp asses +gh ost +Ġins pires +Ġbur geoning +ĠY oshi +ĠVert ical +ĠAud itor +Ġint ending +Ġfilib uster +Bl oom +f ac +ĠCav s +ign ing +Ġcowork ers +ĠBarb arian +rem ember +FL AG +Ġaudit ory +ason ry +Col lege +Ġmut ed +gem ony +ob in +ĠPsych o +9 68 +Ġlav ish +Ġhierarch ical +ĠDr one +ou k +Ġcripp led +ĠMax im +Sl ot +Ġqu iz +ĠV id +if ling +Ġarchae ologists +Ġabandon ment +d ial +le on +ĠF as +T ed +Ġr aspberry +Ġmaneu vers +Ġbehavi ours +Ġins ure +Ġrem od +Sw itch +h oe +Ġsp aced +Ġafford ability +ĠF ern +not ation +ĠBal anced +Ġoccup ies +en vironment +Ġneck lace +Ġsed an +F U +ĠBrav o +Ġab users +ĠAn ita +met adata +ĠG ithub +ait o +ĠF aster +ĠWass erman +ĠF lesh +Ġth orn +r arily +ĠMer ry +w ine +Ġpopul ace +ĠL ann +Ġrepair ing +Ġpsy che +Ġmod ulation +aw aru +âĢĭ âĢĭ +ari j +Ġdecor ations +Ġapolog ise +ĠG arg +app ly +Ġgive away +ĠFl an +ĠWy att +U ber +Ġauthor ised +ĠMor al +HAHA HAHA +activ ate +Ġtorped o +ĠF AR +Ġam assed +ĠA ram +ark in +ĠVict ims +st ab +Ġo m +ĠE CO +Ġopio ids +Ġpurpose ly +ĠV est +Ġer g +at an +ĠSur gery +Ġcorrect ing +ĠOrt iz +ĠBe et +Ġrev oke +Ġfre eway +ĠH iggins +F ail +ĠFar ms +ĠAT P +h ound +Ġp oking +ĠCommun ists +mon ster +iment ary +Ġunlock ing +Ġunf it +we ed +en ario +at ical +ĠEnlight enment +ĠN G +ĠComp ensation +de en +ĠWid ow +ĠCind y +ĠAfter wards +Ġ6 000 +ikh ail +ag ically +Ġrat ified +Ġcasual ty +H OME +p sey +f ee +Ġspark ling +Ġd é +Ġconcert ed +C atal +Ġcomp lying +ĠA res +ĠD ent +Sh ut +Ġsk im +ad minist +Ġhost ilities +ĠG ins +Ġ6 08 +Ġm uddy +ĠMc Int +ĠDec ay +5 25 +Ġconspic uous +ĠEx posure +Ġresc ind +Ġwear able +Ġ3 28 +our met +ah s +ĠRob ots +Ġe clips +inst ance +ĠRE PORT +ĠApp l +0 30 +ĠSk ies +01 00 +Ġfall acy +S ocket +ĠRece iver +Ġsol ves +ĠButter fly +ĠSho pping +ĠFI RE +65 4 +Med ic +Ġsing ers +ĠNeed less +'' '' +isher s +ĠD ive +58 8 +Ġselect ively +Ġcl umsy +88 9 +Ġpurch aser +ear ned +ard y +Ġbenef iting +eng lish +Ġyield ing +ĠP our +Ġspin ach +Ġdel ve +ĠC rom +6 10 +Ġexport ing +ĠMA KE +Ġ26 3 +Ġg rop +Ġenv oy +ĠInqu iry +ĠLu igi +d ry +ĠT uring +Thumbnail Image +ĠVar iety +Ġfac et +Ġfl uffy +Ġexcerpt s +Ġsh orth +ĠOl sen +CL UD +Ġrel iant +ĠUN C +T our +Ġbat hing +Comp any +Ġglobal ization +P red +ĠMalf oy +Ġh oc +j am +craft ed +ĠBond s +ĠKiss inger +Eng land +Ġorder ly +cat entry +Ġ26 1 +Ġexch anging +ĠInt ent +ĠAmend ments +D OM +Ġst out +³³³³³³³³ ³³³³³³³³ +ĠAir bus +Ġ27 8 +hy de +P oll +Item ThumbnailImage +Ġlooph oles +ĠPill ar +Ġexpl or +St retch +A part +Ġun married +Lim it +ĠTransform ers +Ġintellect ually +unct ure +18 00 +Ġd arn +B razil +Ġleft over +ber us +f red +Mine craft +3 26 +ĠForm s +Ġproof s +ĠDes igned +Ġindex es +ĠSupp ose +EM S +ĠL oving +ĠBon nie +im ating +OT US +Ġconduct or +Ġbehav ed +ĠF ren +Ġsy nerg +Ġmillenn ium +Ġcater ing +ĠL auder +W r +ĠY iannopoulos +ĠAT F +Ġensl aved +Ġawaken ed +D VD +ĠED ITION +ĠConc ert +ĠChall enger +ĠH aku +umer ic +Ġdep recated +ĠSH AR +4 12 +Ġdy stop +Ġtremb ling +Ġdread ed +ĠSp ac +p adding +Re pl +ĠG arrison +M ini +Ġun paralleled +am ar +URR ENT +w reck +c ertain +t al +ĠC LS +app ings +Ġsens ed +Ġf encing +ĠPas o +ĠDes k +Ġsc off +Ġcontem plate +ĠL iga +l iquid +75 7 +Ġapp rentice +ĠUCH IJ +5 70 +ĠTh ousand +ĠIll um +Ġchampion ed +ãĤ Į +Ġelect ors +Ġ3 98 +ĠH ancock +round ed +ĠJ OHN +Ġuns atisf +Ġqual ifier +ĠGad get +EN E +Ġdead liest +ĠPl ants +Ġ ions +Ġacc ents +Ġtwe aking +Ġsh aved +F REE +ĠCh aser +Again st +9 60 +Ġmeth amphetamine +Ġnormal ized +Ġ$ \ +ĠPre cision +ĠGu am +Ġch oked +ĠX II +ĠCast ing +Tor rent +Ġscal p +ĠJagu ar +w it +Ġsem ic +ix ie +ĠG ould +Ġconf ines +N usra +ĠL on +ĠJ ugg +y cle +ĠCod ec +E gypt +Ġrest rain +ĠAl iens +Ġch oking +ĠD unk +ĠBell a +ab c +Ġsl ang +Ġneuro trans +s av +Ġempower ment +â ĨĴ +Ġclim bers +ĠM im +ĠF ra +ros se +Cap ital +ĠCth ulhu +Inter face +Ġprof icient +ĠIN TO +Ġ3 18 +ront al +5 80 +ĠDes pair +K enn +Ġscrim mage +ĠCo at +as ions +Ġwall paper +ĠJ ol +Ġresurg ence +Ġant iv +ĠB alls +² ¾ +Ġbuff ers +Ġsub system +ĠSt ellar +ĠL ung +A IDS +Ġerad icate +Ġblat antly +Ġbehav es +ĠN un +Ġant ics +ex port +DE V +w b +Ġph p +ĠInteg rity +Ġexplore r +Ġrev olving +auth ored +g ans +Ġbas k +Ġas ynchronous +å į +TH ING +69 8 +G ene +ĠR acer +ĠN ico +iss ued +Ġser mon +p ossibly +Ġsize of +Ġentrepreneur ial +ox in +ĠMin erva +Ġpl atoon +n os +ri ks +A UT +ĠAval anche +ĠDes c +ij 士 +ĠP oc +Ġconf erred +Î » +Ġpat ched +F BI +66 2 +Ġfract ures +Ġdetect s +Ġded icate +Ġconstitu ent +Ġcos mos +W T +Ġswe ats +Ġspr ung +b ara +s olid +Ġuns us +Ġbul ky +ĠPhilipp e +ĠFen rir +Ġtherap ists +ore al +^^ ^^ +Ġtotal ed +Ġboo ze +ĠR PC +Prosecut ors +Ġdis eng +ĠSh ared +Ġmotor cycles +Ġinvent ions +Ġlett uce +ĠMer ge +ĠJ C +Ġspiritual ity +ĠWAR NING +Ġunl ucky +ĠT ess +Ġtong ues +ĠD UI +T umblr +Ġle ans +Ġinv aders +Ġcan opy +ĠHur ricanes +ĠB ret +ĠAP PLIC +id ine +ick le +Reg arding +Ġve ggies +Ġe jac +ju ven +F ish +D EM +ĠD ino +Th row +ĠCheck ing +be ard +( & +Ġj ails +Ġh r +trans fer +iv ating +Ġfle ets +ĠIm ag +ĠMc Donnell +Ġsnipp et +Is a +ĠCh att +ĠSt ain +ĠSet FontSize +ĠO y +ĠMathemat ics +49 4 +Ġelectro ly +ĠG ott +ĠBr as +B OOK +ĠF inger +d ump +Ġmut ants +Ġrent als +Ġinter tw +Ġc reek +ail a +Bro ther +ĠDisc ord +pe e +raw ler +Ġcar p +Ġ27 9 +ãĤ· ãĥ£ +rel ations +Ġcontr asts +Col umn +Ġrec onnaissance +Ġun know +Ġl ooting +Ġregul ates +Ġopt imum +ĠChero kee +ĠA ry +Lat est +Ġroad side +Ġd anced +ĠUnic orn +A cknowled +Ġuncont roll +ĠM US +at io +ch ance +ha ven +VAL UE +Ġfavour ites +Ġceremon ial +b inary +pe ed +wood s +EM P +Ġv ascular +Ġcontempl ated +Ġbar ren +ĠL IST +Y ellow +ospons ors +Ġwhisk y +ĠM amm +ĠDeV os +min imum +H ung +44 2 +P ic +ĠSnap dragon +77 6 +Ġcar ving +Ġund ecided +Ġadvantage ous +Ġpal ms +ĠA Q +Ġst arch +L oop +Ġpadd le +Ġfl aming +ĠHor izons +An imation +bo ost +Ġprob abilities +ĠM ish +Ġex odus +ĠEditor ial +Ġfung us +Ġdissent ing +ĠDel icious +rog ram +ĠD yn +d isk +t om +Ġfab rics +ĠC ove +ĠB ans +Ġsoft en +ĠCON S +Ġin eligible +Ġestim ating +ĠLex ington +pract ice +of i +Ġshe dding +ĠN ope +Ġbreat hed +ĠCorinth ians +y ne +ek i +B ull +Ġatt aching +reens hots +Ġanaly se +ĠK appa +Ġuns ustainable +Ġinter pol +ank y +he mer +Ġprot agonists +Ġform atted +ĠBry ce +ĠAch illes +ĠAb edin +sh ock +Ġb um +b os +qu a +ĠW arn +q t +ĠDi abetes +8 64 +ĠIn visible +Ġvan ish +Ġtrans mitting +Ġmur ky +ĠFe i +Ġawa ited +ĠJur assic +umm ies +Ġmen acing +g all +C ath +B uilt +ild o +ĠV otes +Ġon t +Ġmun itions +ĠFre em +ÃŃ n +Ġdec ency +lo pp +ie ved +ĠG ord +Ġun thinkable +ĠNews week +Ġ3 21 +He at +Ġpresent er +ji ang +Ġpl ank +ĠAval on +Ġben z +ĠR out +Ġslam ming +ĠD ai +ou ter +ĠCook ie +ĠAlic ia +ge y +Ġvan ity +Ġow l +á µ +t ested +ĠAw akens +Ġcan v +Ġblind ly +ĠRid ley +ĠEm ails +Requ ires +ĠSer bian +ograp hed +if rame +eter ia +Ġaltern ating +qu iet +Ġsoc iology +ĠUn lock +ĠCommun ism +Ġo ps +Ġatt ribution +Ġab duction +ĠAb ram +Ġsidel ined +ĠB OOK +Ġref ining +ĠFe eling +ĠOs lo +ĠPru itt +r ack +ang ible +Ġcaut iously +ĠM ARK +eed s +M ouse +ĠStep h +ĠP air +S ab +99 7 +ĠBa al +B ec +Ġcomm a +ĠP all +ĠG ael +Ġmisunder stand +ĠP esh +Order able +Ġdis mal +ĠSh iny +% " +Ġreal istically +Ġpat io +ĠG w +ĠVirt ue +Ġexhaust ing +wh atever +oph ys +y ip +4 18 +Ad just +ĠWa iting +ess on +ĠMaz da +ĠDo zens +Ġstream lined +Ġincompet ence +ĠM eth +Ġeth os +ON ES +Ġincent iv +Ġgr itty +ĠBut cher +Head er +Ġexp onential +Ã Ł +Ġcorrel ate +Ġcons ensual +s ounding +R ing +Orig in +Ġcon clusive +fe et +ac ly +ĠF ernandez +Buy able +Ġd ucks +aunt lets +Ġel ong +Ġ28 6 +Ġsim ul +G as +ĠK irst +Ġprot r +ĠRob o +ĠAo E +op ol +Ġpsych ologically +sp in +ilater ally +ĠCon rad +W ave +44 1 +ĠAd vertisement +ĠHarm on +ĠOri ental +is Special +Ġpresum ptive +Ġw il +ĠK ier +ne a +Ġp pm +Ġhar bour +ĠW ired +comp any +Ġcor oner +atur days +ĠP roud +ĠN EXT +ĠFl ake +val ued +ce iver +Ġfra ught +Ġc asing +Ġrun away +Ġg in +ĠLaure nt +ĠHar lem +ĠCur iosity +qu ished +Ġneuro science +ĠH ulu +Ġborrow er +Ġpetition er +ĠCo oldown +W ARD +Ġinv oking +conf idence +For ward +Ġst s +pop ulation +Delivery Date +Fil m +ĠC ov +quick Ship +quickShip Available +prim ary +isSpecial Orderable +inventory Quantity +channel Availability +BO X +ĠMulti player +ĠJen ner +77 8 +ĠM d +Ġ~ /. +M N +Ġchild ish +Ġantioxid ant +ĠChrom ebook +Ġ27 4 +Ġscreen play +Ġadvent urous +ĠRelations hip +respons ive +ming ton +Ġcorner stone +ĠF ey +F IR +Ġrook ies +ĠF eaturing +Ġorig inate +Ġelectro des +ant es +Ġscript ures +Ġgl ued +Ġdiscont ent +Ġaff licted +lay out +B rave +Ġm osa +ĠQuant ity +ĠH ik +w inner +H ours +Ġent ail +ĠCell s +olog ue +Ġv il +Ġpre acher +Ġdecor ative +d ifferent +Ġprejud ices +ĠSm oking +ĠNotting ham +so Type +Ġrhyth ms +ĠAl ph +bl ast +Ste el +ĠDaniel le +Ġstr ife +Ġrem atch +so DeliveryDate +ĠF ork +t rip +ol ulu +hes es +C G +ĠPOLIT ICO +ost a +ĠDr ift +é¾įå ¥ +é¾įå¥ ij士 +Ġvet ting +ĠJin ping +ĠRec ession +Min or +ĠF raud +enf ranch +Ġconven ed +ĠNA ACP +ĠMill ions +ĠFarm ing +ĠW oo +ĠFl are +rit o +imm igrant +Ġvac ancy +ĠHE AD +ĠV aj +eg al +ĠV igil +Stud y +Ġru ining +Ġr acks +Ġhe ater +ĠRand olph +ĠBr ush +ĠT ir +Ø ¨ +Ġc ov +% ] +Ġrecount s +ĠO PT +ĠM elt +Ġtr uce +Ġcas inos +Ġcrus ade +Ġcarn age +Ġstri pe +ĠK yl +Text ures +Ġ6 98 +Ġpro clamation +Ġgood ies +Ġ........ .. +pro claimed +P olit +Ġtop ical +Ġspecial ize +ĠA min +g m +Ġanch ored +Ġbear ings +s ample +ĠHigh land +ĠAut ism +Ġmerc enary +Ġinterview er +L ER +ĠSom ers +Ġembry o +ĠAss y +Ġ28 1 +ĠEd iting +ĠCh osen +6 60 +Ġp ci +ĠThunder bolt +BI LL +Ġchuck led +jri wal +h of +Ġearth ly +() { +ind ependence +Ġdisp ers +ĠV endor +ĠG areth +Ġp als +P enn +ĠSub mit +ic um +Th u +Ġcl andestine +Ġcann ibal +ĠCl erk +E Stream +gal itarian +âĻ ¥ +g ew +Ġhor rend +ĠL ov +ĠRe action +ocr in +Class ic +Ġecho ing +Ġdiscl osing +ĠIns ight +og un +ĠInc arn +upload s +pp erc +guy en +Ġ19 01 +ĠB ars +68 7 +Ġb ribes +ĠFres no +ur at +ĠRe ese +Ġintr usive +Ġgri pping +ĠBlue print +ĠR asm +un ia +man aged +ĠHeb do +Ġ3 45 +Ġdec oding +Ġpo ets +Ġj aws +ĠF IGHT +am eless +ĠMead ows +ĠHar baugh +Inter view +ĠH osp +ĠB RA +Ġdelet ion +m ob +W alker +ĠMoon light +ĠJ ed +ĠSoph ia +Ġus ur +Ġfortun ately +ĠPut ting +ĠF old +Ġsan itation +Ġpart isans +IS ON +B ow +ĠCON C +ĠRed uced +ĠS utton +Ġtouch screen +Ġembry os +âĢ¢âĢ¢ âĢ¢âĢ¢ +ĠK rug +com bat +ĠPet roleum +Ġam d +ĠCos mos +Ġpresc ribing +Ġconform ity +ours es +Ġplent iful +Ġdis illusion +ĠEc ology +itt al +Ġf anc +Ġassass inated +regn ancy +Ġperenn ial +ĠBul lets +Ġst ale +Ġc ached +ĠJud ith +ĠDise ases +All en +Ġl as +Ġsh ards +ĠSu arez +ĠFriend ship +inter face +ĠSupp orters +add ons +46 2 +ĠIm ran +ĠW im +Ġnew found +ĠM b +An imal +Ġd arling +and e +Ġrh y +ĠTw isted +pos al +yn ski +Var ious +× ľ +ĠK iw +uy omi +Ġwell being +ĠL au +an os +Ġunm ist +Ġmac OS +Ġrest room +ĠOl iv +ĠAir ways +Ġtimet able +9 80 +Ġrad ios +v oy +ias co +Ġcloud y +ĠDraw ing +Any thing +Sy ria +ĠH ert +st aking +Ġun checked +Ġb razen +ĠN RS +69 7 +onom ic +est ablish +Ġl eng +Ġdi agonal +ĠF ior +L air +ĠSt ard +Ġdef icient +jo ining +be am +Ġomn ip +Ġbl ender +Ġsun rise +Mo ore +ĠF ault +ĠCost ume +ĠM ub +Fl ags +an se +Ġpay out +ĠGovern ors +ĠD illon +ĠBan ana +N ar +Ġtra iled +Ġimperial ist +um ann +ats uki +4 35 +ĠRoad s +Ġsl ur +ĠIde ally +Ġt renches +C trl +Ġmir rored +ĠZ el +ĠC rest +Comp at +ĠRoll s +sc rib +ĠTra ils +omet ers +w inter +Ġimm ortality +il ated +Ġcontrad icts +un iversal +ill ions +ĠM ama +opt im +AT URE +Ġge o +et ter +ĠCar lo +4 24 +Ġcanon ical +ĠStrongh old +n ear +Ġperf ume +Ġorche stra +od iac +Ġup he +Ġreign ing +vers ive +Ġc aucuses +ĠD EM +Ġinsult ed +Ġ---- -- +ĠCr ush +Ġroot ing +ĠWra ith +Ġwh ore +Ġto fu +C md +ĠB ree +Ġ$ _ +Ġr ive +ĠAd vertising +Ġw att +ĠH O +Ġpersu asive +ĠParam eters +Ġobserv ational +ĠN CT +ĠMo j +ĠSal on +Ġtr unc +Ġexqu isite +ĠMar a +Ġpo op +ĠAN N +Ex c +ĠWonder ful +ĠT aco +Ġhome owner +ĠSmith sonian +orpor ated +mm mm +Ġlo af +ĠYam ato +ĠInd o +Ġcl inging +á s +Ġimm utable +h ub +Or ange +Ġfingert ips +ĠWood en +ĠK idd +ĠJ PM +ĠDam n +C ow +c odes +48 2 +Ġiniti ating +ĠEl k +ĠCut ting +Ġabsent ee +ĠV ance +ĠLil ith +G UI +Ġobsc ured +Ġdwar ves +ĠCh op +ĠB oko +Val ues +Ġmult imedia +Ġbrew ed +Reg ular +CRIP TION +ĠMort al +Ġa pex +Ġtravel er +Ġbo ils +Ġspray ing +Rep resent +ĠStars hip +4 28 +Ġdisappro val +Ġshadow y +Ġlament ed +ĠRe place +ĠFran ç +67 7 +d or +Ġunst oppable +Ġcoh orts +gy n +ĠClass ics +ĠAm ph +Ġsl uggish +ĠAdd iction +ĠPad res +Ġins cription +Ġin human +min us +ĠJere miah +at ars +Ter ror +ĠT os +ĠSh arma +ast a +c atch +Ġpl umbing +ĠTim bers +Sh ar +H al +ĠO sc +Ġcou pling +hum ans +Ġsp onge +Ġid ols +ĠSp a +ĠAdv ocate +ĠBe ats +lu a +Ġtick ing +Ġload er +ĠG ron +8 10 +Ġstim ulated +Ġside bar +ĠManufact urer +ore And +19 73 +Ġpra ises +ĠFl ores +dis able +ĠElect rical +ra ise +E th +Ġmigr ated +Ġlect urer +K ids +ĠCa vern +Ġk ettle +Ġgly c +ĠMand ela +ĠF ully +å§ « +FIN EST +Ġsquee zing +ĠRy der +amp oo +oreAnd Online +Inst oreAndOnline +Buyable InstoreAndOnline +Ġcommem orate +ĠRamp age +Aust in +ĠSh roud +ĠRu ins +9 15 +ĠK H +Ġwater front +ĠE SC +b aby +ĠC out +ĠEm blem +Ġequival ents +49 2 +Un ique +ĠNiet zsche +brow ser +Ġim itation +ĠWere wolf +ĠKir in +ac as +' ," +Ġà ¾ +Review ed +Ġc unt +Ġvo ic +ĠLen ovo +Ġbond ed +48 1 +Ġinhib itors +Ġendeav ors +ĠHav ana +ĠSt out +ĠJ olly +A ctor +*/ ( +Ġoccur rences +ĠT ens +Incre ased +ĠACT ION +Ġ ãĢĮ +ĠRank ings +ĠB reat +Ġ30 9 +D ou +Ġimpact ing +ĠDuc hess +pre fix +Q B +Ġsummon ing +Ġbest owed +ĠKe pler +ĠPOW ER +c ube +ĠK its +ĠG rip +Ġop ium +Ġrep utable +t oc +ich ael +ĠR ipple +Ġcaf é +ĠZ oom +ĠBur ma +Ġwa ive +Ġst alls +Ġdem eanor +inc erity +Ġfluor ide +ĠSH OULD +Par is +Ġlong ing +Ġpl at +Ġgross ly +Ġbull s +Ġshowc asing +ex pected +ĠG addafi +engine ering +Re peat +ĠK ut +Ġconce ivable +Ġtrim med +osc ope +ĠCand idate +ĠT ears +rol og +Lew is +S UP +Ġroad map +Ġsal iva +Ġtrump et +Jim my +Ġmirac ulous +Ġcolon ization +Ġam put +ĠGN OME +ate ch +D ifferent +ĠE LE +ĠGovern ments +ĠA head +ãħĭ ãħĭ +word press +L IB +ĠIn clude +ĠDor othy +0 45 +ĠColomb ian +Ġle ased +88 4 +Ġde grading +ĠDa isy +i ations +Ġbapt ized +Ġsurn ame +co x +Ġblink ed +ãĥ ¢ +Ġpoll en +Ġder mat +Ġre gex +ĠNich olson +ĠE ater +ç ľ +rad or +Ġnarrow er +Ġhur ricanes +Ġhalluc inations +r idden +ISS ION +ĠFire fly +Ġattain ment +Ġnom inate +Ġav ocado +ĠM eredith +Ġt s +Ġreve rence +Ġe uph +Ġcr ates +ĠT EXT +Ġ4 43 +Ġ3 19 +J SON +iqu ette +Ġshort stop +ic key +Ġpro pelled +Ġap i +ĠTh ieves +77 9 +Ġovers aw +Ġcol i +ĠNic ola +Ġover cl +ik awa +ĠC yr +Ġ38 4 +78 9 +ĠAll ows +10 27 +Det roit +TR Y +set up +ĠSocial ism +Sov iet +s usp +ĠAP R +ĠShut down +Ġal uminium +zb ek +ĠL over +GGGG GGGG +Ġdemocr acies +Ġ19 08 +ĠMer rill +ĠFranco is +gd ala +Ġtraff ickers +ĠT il +ĠGo at +Ġsp ed +ĠRes erv +Ġpro d +55 2 +Ġc ac +ĠUn iv +ĠSch we +Ġsw irling +ĠWild erness +ĠEgg s +Ġsadd ened +Ġarch aic +H yd +Ġexcess ively +B RE +Ġaer ospace +ĠVo ices +Cra ig +Ġign ited +In itially +ĠMc A +Ġhand set +Ġreform ing +Ġfrust rations +ĠDead pool +ĠBel ichick +ract or +ĠRagnar ok +ĠD rupal +ĠApp roximately +19 20 +ĠHub ble +arm or +ĠSar as +ĠJon as +Ġnostalg ic +Ġfeas ibility +Sah aran +Ġorb iting +Ġ9 70 +R u +Ġsh in +ĠInvestig ators +Ġinconsist encies +ĠP AN +B G +Ġgraz ing +Ġdetect ors +ĠStart up +ĠFun ny +ĠNa omi +Consider ing +Ġh og +ut f +ce mic +Ġfort ified +ĠFun ctions +Ġcod ec +nut rition +H at +" ! +micro soft +55 8 +ĠTh in +ĠA CE +Al ias +ĠO PS +p apers +P K +ãĢ İ +Ġimpro bable +N orthern +equ al +Ġlook out +Ġty res +ĠMod ified +ĠK op +Abs olutely +Ġbuild up +sil ver +Ġaud i +Ġgro tesque +ĠSab er +ĠPres byter +ON Y +Ġglac iers +ĠSho als +ĠK ass +ĠH RC +ĠNic ol +ĠL unch +ĠF oss +âĸ Ĵ +AD RA +ĠOne Plus +o ing +ground s +Ġincident al +Ġdatas ets +68 9 +ĠClarks on +Ġassemb ling +ĠCorrect ions +Ġdrink ers +Ġqual ifiers +Ġle ash +Ġunf ounded +ĠH undred +Ġkick off +T i +Ġrecon cil +ĠGr ants +ĠCompl iance +ĠDexter ity +Ġ19 06 +w arn +D allas +Max imum +n ard +av ia +be aut +ens itivity +tr ace +Ġpione ers +ĠF ract +ãĢ ı +Ġpre cept +Ġgloss y +ĠI EEE +Ac ross +Ġ6 80 +S leep +che on +Ġsatir ical +ĠMin otaur +ĠCla ude +Ġr é +ape go +Ġcar rot +ĠSem in +ino a +Ġz o +Ind ependent +Ġdiagn oses +ĠC ue +M AR +Ġrend ition +ĠK ik +Ġpath ology +Ġselect s +Link edIn +Ġass ay +ĠD res +Ġtext ual +post ed +IT AL +ĠM aul +N eal +Ġinter connected +Ġerr atic +ĠVir us +Ġ5 30 +Ġenvironmental ists +ĠP helps +Ġeng agements +ĠIN ST +Ġeconom ical +nox ious +Ġg earing +izz y +Ġfavor ably +ĠMcG ill +T erm +Ġh anged +Ġball park +ĠRe yes +Ġbe ware +ĠP sal +ĠMass acre +q i +Ġin accessible +acly sm +Ġfr ay +ill ac +Ġbitter ly +ĠCert ification +Mich igan +Ġir respective +al ore +Em pty +Ġendorse ments +Ġund et +f g +equ ipped +Ġmerc iless +ĠC ust +Ġimm ature +Ġvou cher +ĠBlack well +Ñ ı +h awk +dis ciplinary +ile e +ĠMak oto +ĠD ude +ãĥĩ ãĤ£ +Y ears +Ġin ver +Ġsh aman +ĠY ong +ip el +ell en +ĠCath y +br ids +Ġs arc +65 1 +N ear +Ġground work +Ġam az +Ġ4 15 +ĠHunting ton +hew s +ĠB ung +Ġarbit rarily +ĠW it +ĠAl berto +Ġdis qualified +best os +46 1 +Ġp c +Ġ28 4 +ro bat +Rob in +Ġh ugs +ĠTrans ition +ĠOcc asionally +Ġ3 26 +ĠWh ilst +ĠLe y +Ġspaces hip +cs v +Ġun successfully +ĠA u +le ck +ĠWing ed +ĠGrizz lies +. � +Ġne arer +ĠSorce ress +ĠInd igo +El se +8 40 +let es +Co ach +Ġup bringing +ĠK es +Ġseparat ist +Ġrac ists +Ġch ained +Ġabst inence +lear ning +Ġrein stated +Ġsymm etry +Ġremind ers +ĠChe vy +Ġm ont +Ġexempl ary +ĠT OR +Z X +Ġqual itative +ĠSt amp +ĠSav annah +ĠRoss i +Ġp aed +Ġdispens aries +ĠWall s +ĠCh ronic +Ġcompliment ary +ĠBeir ut +Ġ+ --- +igs list +Ġcrypt ographic +mas ters +ĠCap itals +Ġmax imal +Ġent ropy +Point s +Ġcombat ants +l ip +ĠGl ob +ĠB MC +ph ase +th ank +HT TP +Ġcomm uter +Ġ\( \ +.. / +ĠReg ener +ĠDO I +ĠActiv ision +Ġsl it +os al +RE M +Ġch ants +Y u +Ke ys +Bre xit +ĠFor ced +Ari zona +Ġsquad ron +IS O +ĠMal one +Ġ3 38 +Ġcontrast ing +Ġt idal +Ġlib el +Ġimpl anted +Ġupro ar +ĠC ater +Ġpropos itions +M anchester +ĠEuro s +it amin +G il +ĠEl ven +ĠSe ek +ĠB ai +Ġredevelop ment +ĠTown s +ĠL ub +! ", +al on +K rist +Ġmeas urable +Ġimagin able +Ġapost les +Y N +7 60 +Ġster oid +Ġspecific ity +ĠL ocated +ĠBeck er +ĠE du +ĠDiet ary +uts ch +ĠMar ilyn +Ġbl ister +ĠM EP +ĠK oz +ĠC MS +y ahoo +ĠCar ney +Ġbo asting +ĠC aleb +By te +read s +ad en +Pro blem +ĠWood ward +S we +S up +ĠK GB +Set up +Ġtac it +Ġret ribution +Ġd ues +ĠM ü +. ? +ä¸ Ń +p ots +Ġcame o +ĠP AL +educ ation +A my +like ly +g ling +Ġconstitution ally +ĠHam m +ĠSpe ak +Ġwid gets +br ate +Ġcra ppy +ĠI ter +Ġanticip ating +ĠB out +P ixel +ĠY ep +ĠLaur ie +Ġh ut +Ġbullet in +ĠSal vation +Ġch ats +ear able +Honest ly +AL TH +onse qu +c ult +isco very +ovy ch +Ġse lves +ĠSat oshi +S ounds +Ġconver gence +ĠRosen berg +19 74 +Ġnas al +Ġfull est +Ġfer ocious +x us +ist e +AM S +Ġlobb ied +Ġso othing +ĠGun n +t oday +0 24 +Ġinspir ational +ĠN BN +p b +g ewater +or ah +all owed +ĠCol iseum +Ġspecial izing +Ġinsane ly +ĠT ape +del ay +Ġt arn +ĠP ound +Ġmel anch +Ġdeploy ments +il and +Ġless en +Ġfur ry +ĠUE FA +Ġblood shed +ĠMe ier +ither ing +Ġhe irs +ĠJ aw +ax ter +ĠPublic ations +Ġal ters +int ention +ĠWinc hester +d etermination +ĠLif etime +th in +Mon ster +7 80 +Ġapprox imation +Ġsuper markets +ĠSecond s +or os +h uge +Ġb ribe +ĠLIM ITED +un ed +Ġmis interpret +ĠIn jury +Ġ3 67 +Ġthreshold s +ĠCarn ival +Ġgastro intestinal +Ġguid eline +Ġde ceived +f eatures +Ġpurported ly +ĠRon nie +ĠNew t +Ġsp acious +as us +Ġsuperhero es +ĠCyn thia +le gged +k amp +ch io +Ġth umbnail +ĠShir ley +ill ation +Ġshe ds +ĠZ y +E PA +Ġdam s +Ġy awn +n ah +ĠPe ggy +ĠE rie +ĠJu ventus +ĠF ountain +r x +don ald +al bum +ĠComp rehensive +Ġc aching +ĠU z +ulner ability +ĠPrinc iple +ĠJ ian +ing ers +cast s +ĠOs iris +ch art +t ile +ĠTiff any +ĠPatt on +ĠWh ip +Ġovers ized +J e +ĠCind erella +ĠB orders +ĠDa esh +M ah +Ġdog ma +Ġcommun ists +v u +Coun cil +Ġfresh water +Ġw ounding +Ġdeb acle +Ġyoung ster +Ġthread ed +ĠB ots +ĠSav ings +ãģ Ĥ +ol ing +oh o +Ġillum ination +M RI +Ġlo osen +tr ump +ag ency +ur ion +Ġmoment arily +ĠCh un +ĠBud apest +ĠAl ley +D isk +Ġaston ished +ĠCon quer +ĠAccount ing +h aving +ĠWe in +ĠAl right +Ġrev olver +Ġdel usion +Ġrelic s +Ġad herent +qu ant +Ġhand made +or io +Ġcomb ating +c oded +Ġquad ru +re th +N ik +ĠTrib al +ĠMyster ious +Ġin hal +ĠWin ning +ĠClass ification +ch anged +Ġun ab +Ġsc orn +icip ated +w l +ond uctor +Ġrein forcing +ĠChild hood +an ova +Ġadventure r +Ġdoctor al +ĠStrateg ies +Ġengulf ed +ĠEnc ounter +Ġl ashes +Crit ical +ric ular +ĠU TF +oci ation +check ing +ĠConsult ing +Run time +per iod +ĠAs gard +Ġdist illed +ĠPas adena +ĠD ying +ĠCOUN TY +Ġgran ite +Ġsm ack +Ġparach ute +ĠS UR +Virgin ia +ĠF urious +78 7 +ĠO kin +Ġcam el +ĠM bps +19 72 +ĠCh ao +ĠC yan +j oice +ef er +ĠW rap +ĠDeb ate +S eg +Ġfore arm +ĠIgn ore +Ġtim estamp +Ġprob ing +ĠNo on +ĠGra il +f en +Ġdorm ant +ĠFirst ly +ĠE ighth +ĠH UN +ĠDes ire +or as +Girl s +ĠDes mond +z ar +am ines +O AD +exec ute +Ġbo obs +ĠAT L +_ ( +Chel sea +Ġmasturb ation +ĠCo C +Ġdestroy er +ĠCh omsky +Ġsc atter +ĠAss ets +79 6 +ĠC argo +Ġrecept ive +ĠSc ope +Ġmarket ers +Ġlaun chers +Ġax le +ĠSE A +se q +ĠM off +f inding +ĠGib bs +Georg ia +extreme ly +N J +Ġlab orers +st als +Ġmed iation +ĠH edge +at own +Ġi od +des pite +v ill +J ane +ex istence +Ġcoinc ided +ĠUt ilities +ĠChe ap +Ġlog istical +Ġcul mination +ĠNic otine +p ak +F older +Ġrod ents +st uff +Ġlaw fully +Ġreper to +io ch +j j +Dial ogue +HH HH +lic tion +Look s +Ġ29 7 +Ġtur rets +ĠAb andon +Ġinc ess +ĠTraff ord +Ġcur led +Ġprefer ring +Ġprivat ization +Ġir resist +ĠP anda +ĠSh ake +ĠMc Gr +ãĥ Ħ +und ers +Ġdiscrim inated +Ġbart ender +I LE +Atl antic +Ġprop ensity +ĠW iz +ĠG im +con ference +Ġrein forces +G h +w agon +Ġe erie +F al +Ġhug ged +rac ist +R IC +F u +Ġf iller +ĠSt ub +Ġeng raved +ĠWrest le +Ġimagin ative +ĠPe er +ĠFact ors +an us +ĠDrac ula +mon itor +Ġrou ters +ib ia +ĠBoo lean +end ale +ĠSl aughter +ĠSh ack +R FC +ĠSpiel berg +S ax +ĠPH OTO +ĠCl over +ĠR ae +Dep ending +ĠMem or +ar am +Ġpier ced +Ġcur tains +v ale +ĠInqu isition +ĠP oke +Ġforecast ing +Ġcompl ains +S ense +ĠHer mes +isc overed +Ġb ible +ĠMor ph +Ġg erm +78 5 +D ON +Ġcon gen +Ġcr ane +ĠD PR +Ġrespect fully +R oom +ĠN aw +ĠDal ai +re ason +ĠAng us +Educ ation +ĠTitan ic +Ë ľ +Ġo val +un ited +Ġthird s +Ġmoist ur +ĠC PC +M iami +Ġtent acles +ĠPol aris +ex c +ex clusive +ĠPra irie +Ġcol ossal +ĠBl end +sur prisingly +ÃŃ s +Ġindo ctr +Ġbas al +ĠMP EG +und o +Spl it +Develop ment +Ġlan tern +19 71 +Ġprov ocation +Ġang uish +ĠB ind +ĠLe ia +duc ers +ipp y +conserv ancy +Ġinitial ize +ĠTw ice +ĠSu k +Ġpred ic +Ġdi ploma +Ġsoc iop +Ing redients +Ġhamm ered +ĠIr ma +Q aida +Ġglim ps +ĠB ian +Ġst acking +Ġf end +gov track +Ġun n +dem ocratic +ig ree +Ġ5 80 +Ġ29 4 +Ġstraw berry +ID ER +Ġcher ished +ĠH ots +Ġinfer red +Ġ8 08 +ĠS ocrates +O regon +ĠR oses +ĠFO IA +Ġins ensitive +Ġ40 8 +Recomm end +ĠSh ine +Ġpain staking +UG E +ĠHell er +ĠEnter prises +I OR +ad j +N RS +L G +Ġalien ated +Ġacknowled gement +ĠA UD +ĠRen eg +Ġvou chers +Ġ9 60 +Ġm oot +ĠDim ensions +Ġc abbage +B right +g at +ĠK lu +Ġlat ent +Ġz e +ĠM eng +Ġdis perse +Ġpand emonium +H Q +Ġvirt uous +ĠLoc ations +ee per +prov ided +Ġse ams +ĠW T +iz o +PR OV +Ġtit anium +Ġrecol lection +Ġcr an +Ġ7 80 +ĠN F +49 1 +64 2 +p acking +59 8 +text ure +Sp ider +fre edom +cipl ed +ĠTAM ADRA +âĻ ¦ +aut hent +ĠW ANT +r ified +Ġr ites +Ġuter us +k iss +Ġâī ¤ +Ġsk illet +Ġdis enfranch +ĠGa al +Comp an +Ġage ing +gu ide +B alt +Ġiter ator +Ġdiscretion ary +t ips +Ġprim ates +ĠTechn ique +ĠPay ments +az el +ĠR OCK +stant ial +0 60 +Ġd mg +ĠJack ets +ĠPlay off +Ġnurs ery +ĠSy mb +art on +Ġannex ation +Color ado +Ġco ils +ĠSh oes +âĦ¢ : +ĠRo z +COM PLE +ĠEve rest +ĠTri umph +J oy +G rid +à ¼ +process or +ĠPros per +ĠSever us +ĠSelect ed +r g +ĠTay yip +St ra +Ġski ing +Ġ? ) +Ġpe g +Tes la +Ġtime frame +Ġmaster mind +ĠN B +scient ific +ĠSh it +gener ic +IN TER +N UM +Ġst roll +ĠEn ix +ĠM MR +ĠE MS +m ovie +Ĥ ª +Ġminim izing +idd ling +Ġilleg itimate +Ġprot otyp +Ġpremature ly +Ġmanual s +obb ies +ĠCass idy +D EC +des ktop +Ġaer os +Ġscreen ings +Ġdeb ilitating +ĠGr ind +nature conservancy +Ġf ades +ter mination +assets adobe +F actor +Ġdefinitive ly +P oké +ap ult +ĠLaf ayette +C orn +ĠCor al +Ġstagn ant +T ue +Ġdissatisf action +G ender +Ġkid neys +ĠG ow +ĠDef eat +ĠAsh ton +Ġcart els +Ġfore closure +ĠExpl ore +stre ngth +ot in +Ġveterin arian +Ġf umble +Ġpar ap +ĠSt rait +r ils +Ġpr ick +ĠBerm uda +ĠAm munition +skin ned +Ġab ound +ĠB raz +Ġshar per +ĠAsc ension +Ġ9 78 +Ġpreview s +Ġcommun ion +ĠX Y +Ġph ony +Ġnewcom er +Ġ3 32 +." ," +Ġredist ribution +Prot ect +ĠSo f +K al +Ġlip stick +w orst +Ġtang led +Ġretrospect ive +int eger +Ġvolunte ering +Ġ19 07 +Ġ -------------------- +ic hen +Ġunve iling +Ġsen seless +Ġfisher ies +\ - +Ġh inges +Ġcalcul us +My th +Ġund efeated +Ġoptim izations +Ġdep ress +Ġbill board +ĠY ad +ĠPy ramid +Is n +I de +Ġleg ion +ĠK ramer +ent anyl +Ġpenet rating +ĠHaw th +ĠPR ODUCT +ĠGer ard +ĠP act +ĠIn cluding +ĠEl ias +ĠEl aine +vis ual +Ġhum ming +Ġcond esc +ĠF asc +ä¸ Ĭ +Ġe galitarian +Ġdev s +ĠD ahl +O ps +D H +ĠB ounce +id ated +ald o +Ġrepublic an +Ġh amb +ĠS ett +ograph ies +CH APTER +Ġtrans sexual +Ġsky rocket +ans wer +Ġmark up +Ø ª +Ġhero ine +Comp are +ĠT av +Be ast +Ġsuccess ors +Ġna ïve +ĠBuck ley +st ress +me at +Ġdownload able +Ġindex ed +Ġsc aff +ĠL ump +ĠHom o +Stud io +In sp +Ġr acked +far ious +ĠPet ty +Ex ternal +Ġ19 09 +W ars +com mit +put ers +Ġun ob +ĠEr r +ĠE G +ĠAl am +ĠSiber ia +ĠAtmosp heric +IS TER +ĠSatan ic +trans lation +ĠL oud +tra umatic +l ique +Ġreson ate +ĠWel ch +Ġspark ing +ĠT OM +t one +Ġout l +Ġhandc uffed +ĠSer ie +8 01 +Ġland marks +ĠRee ves +Ġsoft ened +Ġdazz ling +ĠW anted +month s +Mag ikarp +Ġunt reated +ĠBed ford +M i +ĠDynam o +O re +79 5 +Ġwrong ful +Ġl ured +Ġcort isol +Ġve x +d rawn +ile t +Download ha +ĠF action +Ġlab yrinth +Ġhij acked +w aters +er ick +Ġsuper iors +ĠRow ling +ĠGu inness +Ġt d +99 2 +Ġune arthed +Ġcentr if +Ġsham eless +P od +ĠF ib +Ġ icing +Ġpredict or +Ġ29 2 +fore station +con struct +C and +@ # +Ġag itated +Ġre pr +OV A +Ġkn itting +ĠLim a +Ġf odder +68 4 +ĠPerson a +k l +7 01 +Ġbreak up +á ¸ +Ġapp alled +Ġantidepress ants +ĠSus sex +Har ris +ĠTher mal +ee ee +U pload +Ġg ulf +Ġdoor step +ĠSh ank +L U +ĠM EN +ĠP ond +s orry +Ġmis fortune +n ance +Ġb ona +M ut +Ġde graded +ĠL OG +ĠN ess +an imal +Ġa version +und own +Ġsupplement ed +ĠC ups +Ġ50 4 +Ġdep rive +ĠSpark le +Å Ĥ +ĠMed itation +auth ors +ĠSab an +ĠN aked +air d +ĠMand arin +ĠScript ures +ĠPerson nel +ĠMahar ashtra +Ġ19 03 +ĠP ai +ĠMir age +omb at +Access ory +Ġfrag mented +T ogether +Ġbelie vable +ĠGl adiator +al igned +ĠSl ug +M AT +Ġconvert ible +ĠBour bon +amer on +ĠRe hab +nt ax +Ġpowd ered +pill ar +Ġsm oker +ĠMans on +ĠB F +5 11 +ĠGood ell +ĠD AR +m ud +g art +Ġob edient +ĠTrans mission +ĠDon ation +8 80 +Ġbother ing +Material s +ãĤ ± +dest roy +Ġfore going +Ġanarch ism +ĠK ry +ice ps +Ġl ittered +ĠSch iff +Ġanecd otal +un its +Ġf ian +ĠSt im +ĠS OME +ĠInv aders +Ġbehaviour al +ĠVent ures +Ġsub lime +Ġfru ition +ĠPen alty +Ġcorros ion +¶ ħ +Ġlik ened +Ġbesie ged +ween ey +ĠCre ep +Ġlinem en +mult i +ic ably +ud der +Ġvital ity +Ġshort fall +ĠP ants +ap ist +H idden +ĠDro ps +med ical +Ġpron unciation +ĠN RL +Ġinsight ful +J V +ĠBe ard +ĠCh ou +Ġchar ms +Ġb ins +Ġamb assadors +ĠS aturdays +Ġinhib itor +ĠFr anch +6 01 +', ' +ĠCon or +art ney +ĠX peria +g rave +be es +ĠProtest ants +Ġso aking +ĠM andal +Ġph ased +Ġ6 60 +Ġsc ams +Ġbuzz ing +ĠItal ians +ĠLoren zo +ĠJ A +Ġhes itated +Ġcl iffs +ĠG OT +ingu ishable +Ġk o +Ġinter ruption +Z ip +Lear ning +Ġundersc ores +ĠBl ink +K u +57 9 +ĠAut ob +I RE +Ġwater ing +Ġpast ry +8 20 +Ġvision ary +ĠTempl ar +awa ited +Ġpist on +Ġant id +current ly +Ġp ard +Ġw aging +Ġnob ility +ĠY us +Ġinject ing +f aith +ĠP ASS +å º +Ġret ake +ĠPR OC +Ġcat hedral +b ash +Ġwrest lers +Ġpartner ing +Ġn oses +Ġ3 58 +Trans form +am en +Ġb outs +ĠId eal +ĠConstant in +Ġse p +ĠMon arch +att en +ĠPe oples +mod ified +Ġmor atorium +Ġpen chant +Ġoffensive ly +Ġprox ies +ok ane +ĠTaiwan ese +ĠP oo +ĠH OME +us ional +Ġver bs +ĠO man +vis ory +Ġpersu asion +Ġmult it +Ġsc issors +G ay +ow ay +oph ysical +l us +gn u +Ġap ocalyptic +Ġabsurd ity +Ġplay book +Ġautobi ography +I UM +Ġsne aking +ĠSim ulation +pp s +ell ery +Plan et +Ġright fully +Ġn iece +ĠN EC +ĠIP O +ĠDis closure +lean or +ous y +ST ER +Ġ28 2 +Cru z +Ch all +64 3 +ĠSurv ive +ĠF atal +ĠAm id +ap o +We apons +D EN +7 70 +ĠGreen wald +Ġlin en +al os +Ġpollut ants +ĠPCI e +k at +Ġp aw +ĠK raft +C hem +ĠTermin ator +Ġre incarn +Ġ] [ +ĠSe eds +Ġsilhou ette +ĠSt ores +Ġgro oming +ĠD irection +ĠIs abel +ĠBr idges +ðŁ ij +E ED +ĠM orsi +Ġval ves +ĠRank ed +ĠPh arma +ĠOrgan izations +Ġpenet rated +ĠRod ham +ĠProt oss +Ġove rest +Ġex asper +ĠT J +Ġ 000000 +Ġtrick le +Ġbour bon +WH O +Ġw retched +Ġmicrosc opic +Ġcheck list +Ġad orned +R oyal +Ad minist +ĠRet irement +ĠHig hest +We ather +ile ge +Ġincre ments +ĠC osponsors +Ġmas se +ĠS inn +r f +Ġh ordes +as sembly +75 4 +ĠNat asha +ĠTY PE +ĠGEN ERAL +Ġarr anging +Ġ40 7 +l ator +Ġg lean +Ġdisc redited +Ġclin icians +UN E +Ġachie ves +ĠEm erson +com plex += [ +Ġprincip ally +Ġfra il +p icked +Ġthan king +Ġre cl +ĠL AST +Ġsupp ressing +il ic +Ġantidepress ant +ĠLis bon +Ġth or +Ġsp a +Ġking doms +ĠPear ce +em o +Ġpl ung +Ġdiv est +Ġ ******************************** +b is +osp els +ad r +Sp irit +hall a +P ink +end ez +Ġresurrect ed +esc ape +ĠRosen stein +Ġge ological +Ġnecess ities +Ġcarn iv +ĠE lys +ĠBar ney +Ġ29 6 +dig y +ST ON +D OWN +Ġmil estones +Ġk er +Ġdismant ling +Ġre prim +Ġcross ings +19 45 +Ġpatri archy +Ġblasp hemy +Ġ3 59 +met ry +ĠOb esity +ĠDiff erences +bl ocking +ãĥķ ãĤ¡ +ich ita +ĠSab ha +ph alt +ĠCol o +ual a +effic ients +ĠMed ina +con sole +55 7 +ĠHann ibal +ĠHab it +ĠF ever +Ġthen ce +Ġsyn agogue +Ġessential s +Ġw ink +ĠTr ader +ID A +ĠSp oiler +ĠIceland ic +ĠHay ward +Ġpe ac +Ġmal ice +Ġflash back +Ġth w +Ġlay offs +L iquid +Ġtro oper +Ġh inge +ĠRead ers +Ph ill +ĠB auer +Cre ated +Ġaud its +ac compan +Ġunsus pecting +ier a +6666 6666 +Ġbro ch +Ġapprehend ed +ĠM alk +cer ning +ĠCod ex +O VER +M arsh +ĠD eng +ĠExp ression +Ġdisrespect ful +Ġasc ending +t ests +ĠPlaint iff +ster y +ĠAl ibaba +din and +ĠDem psey +Applic ations +mor al +Ġthrough put +Ġquar rel +Ġm ills +Ġhe mor +ĠC ASE +terror ist +st im +ifest yle +ro zen +CE PT +Ar k +u ci +lect ic +Ġirrit ating +she ets +A y +Ġrede emed +Ġhorn y +ĠTe ach +ĠS ear +dem ocracy +4 65 +ĠRest ore +Ġstand by +ĠP is +iff in +Ġsleep y +Ġextr ater +Ġcompl iments +Fram eworks +Ġinstall s +Ġb anging +sur face +found land +Ġmetaph ysical +Ġ28 3 +oul s +dev ices +Ar gs +ĠSac rifice +ĠMcC orm +es on +Cons ervative +ĠM ikhail +see ing +is ively +ĠRo oms +ĠGener ic +Ġenthusi astically +Ġgri pped +Ġcomed ic +ĠElectric ity +Ġgu errilla +Ġdec oration +ĠPerspect ive +Ġconsult ations +Ġun amb +Ġplag iar +Ġmagic ian +Ġe rection +ĠTour ism +or ied +ro xy +11 00 +T am +Ī è +Î ³ +× ª +ĠPred ators +Nit rome +Ġtelesc opes +project s +Ġun protected +Ġst ocked +ĠEnt reprene +nex pected +Ġwast ewater +V ill +Ġint imately +Ġi Cloud +ĠConst able +Ġspo of +Ġne farious +Ġfin s +Ġcens or +ĠMod es +ĠEs per +ar bon +Ġinter sections +Ġlaud ed +Ġphys i +Ġgener ously +ĠThe Nitrome +ĠTheNitrome Fan +Ġar isen +ĠÙ Ī +Ġg lands +ĠPav ilion +ĠGu pta +Ġuniform ly +Ġr amps +ri et +ĠWH EN +ĠVan essa +Ġrout ed +Ġlim p +ĠC PI +p ter +int uitive +Ġv aping +Ġexperiment ed +ĠOlymp us +ĠAm on +Ġsight ing +Ġinfiltr ate +ĠGentle man +Ġsign ings +ĠMe ow +ĠNav igation +che cks +4 33 +Ġel apsed +ĠBulg arian +esp ie +ĠS OM +d uring +Ġsp ills +anc a +ĠPly mouth +M AL +Ġdomest ically +ĠWater gate +ĠF AM +k illed +ed ited +ĠYour self +Ġsynchron ization +ĠPract ices +ST EP +Ġgen omes +ĠQ R +not ice +Ġloc ating +z in +Ġ3 29 +al cohol +Ġk itten +V o +Ġr inse +Ġgrapp le +ĠSc rew +ĠD ul +A IR +Ġle asing +ĠCaf é +Ġro ses +ĠRes pect +Ġmis lead +Ġperfect ed +Ġnud ity +Ġnon partisan +ĠCons umption +Report ing +Ġnu ances +Ġdeduct ible +ĠSh ots +Ġ3 77 +Ġæ ľ +ano oga +Ben ef +ĠB am +ĠS amp +if ix +Ġgal van +ĠMed als +rad ius +Ġno bles +Ġe aves +igr ate +K T +ĠHar bour +u ers +Ġrisk ed +re q +Ġneuro t +get table +ain a +Rom ney +Ġunder pin +Ġlo ft +ĠSub committee +ĠMong ol +b iz +Ġmanif ests +ass isted +ĠG aga +Ġsy nergy +Ġreligious ly +ĠPre f +ĠG erry +T AG +ĠCho i +4 66 +beh ind +ĠO u +Gold Magikarp +Ġhemor rh +R iver +Ġtend on +Ġinj ure +ĠF iona +Ġp ag +Ġag itation +|| || +ur an +ĠE SA +Ġest eem +Ġdod ging +Ġ4 12 +r ss +Ġce ases +ex cluding +Ġint akes +Ġinsert s +Ġemb old +ĠO ral +up uncture +4 11 +ĠUn ified +ĠDe le +Ġfurn ace +ĠCoy otes +ĠBr ach +L abor +Ġhand shake +Ġbru ises +Gr ade +éĹ ĺ +ĠGram my +ile en +St ates +ĠScandinav ian +ĠKard ash +8 66 +Ġeffort lessly +ĠDI RECT +ĠTH EN +ĠMe i +ert ation +19 68 +Ġgro in +w itch +Requ irements +98 5 +Ġroof s +Ġest ates +ĠH F +Ġha ha +Ġdense ly +ĠO CT +Ġpl astics +Ġincident ally +ĠTr acks +ĠTax es +Ġch anted +Ġforce ful +ĠBie ber +ĠK ahn +K ent +ĠC ot +lic ts +F ed +Ġhide ous +ĠVer d +ĠSynd icate +ĠIl legal +J et +ĠD AV +re asonable +c rew +Ġfundamental ist +Ġtruth ful +ĠJ ing +Ġl il +Ġdown ed +Ġen chanted +ĠPolic ies +ĠMcM aster +ĠH are +ides how +Ġpar ams +en cers +gorith m +Ġallow ances +Ġturb ulent +Ġcomplex ities +ĠK T +Ġ3 37 +ĠGen etic +F UN +D oug +t ick +Ġg igs +ument hal +Ġpatriarch al +Ġcal c +, ... +Ġc out +ĠGu an +Ġpath ological +ĠR ivals +Ġunder rated +Ġflu orescent +ĠJ iu +arna ev +ĠQu an +Ġ4 29 +Ġ ਠ+M ario +Con struct +ĠC itation +ĠR acial +ĠR SA +ĠF idel +Ġ3 95 +Person ally +C ause +à » +rad ical +in en +Ġvehement ly +ĠPap a +Ġintern ship +Ġfl akes +ĠRe ck +Luck ily +B ra +20 20 +rav ings +R N +W onder +Ser iously +Ġre usable +Ġpoll uted +ĠP eng +le igh +ind le +Ġcircuit ry +ĠMad onna +ĠB ART +Res idents +att ribute +Phil adelphia +Cl ub +Ġplan ner +Ġfr antically +Ġfaith fully +ĠTerrit ories +ĠL AT +ĠAnders en +an u +ĠP ARK +ĠS ora +i age +ĠPlay offs +ĠG CC +4 27 +Ġab norm +ĠL ever +Ġdisob edience +As ync +ĠShe a +V ert +Ġsk irts +ĠSaw yer +x p +Ġwors ening +Ġsc apego +ĠAng le +oth al +Ġtro ve +ĠSt y +ĠN guyen +mar ine +ide on +Dep ths +Bl og +ĠIll uminati +Ġtract s +Ġorgan ise +Ġo str +F s +Ġlever aging +ĠD aredevil +as ar +Ġl ang +Ġex termin +urs ions +ĠRom o +ãĤ¤ ãĥĪ +Ġcont ended +Ġencounter ing +ĠTable t +ĠAltern ate +sk ill +Ġswe ets +Ġco hesive +cap acity +Ġrep ud +Ġl izard +ro o +Ġpilgr ims +ĠR uff +ĠInstr ument +ĠLog o +uit ous +E H +Ġsales man +Ġank les +L ed +ĠPat ty +ud os +Own er +Ġdiscrep ancies +k j +M U +Ġuncond itional +Dragon Magazine +i ard +O ak +ĠConvers ation +be er +ĠOs aka +D elta +us ky +Ġsecret ion +Ġpl aza +Ġm ing +Ġde pletion +ĠM ous +ĠI TS +ĠH imal +ĠFle ming +Ġcyt ok +ĠH ick +Ġbat ters +ĠInt ellectual +6 75 +é r +IS ION +ĠQu entin +ĠCh apters +ih adi +Ġco aster +WAY S +ĠL izard +ĠY or +and ering +S kin +ha ust +ab by +Ġportray ing +Ġwield ed +d ash +Ġprop onent +Ġr ipple +Ġgrap hene +Ġfly er +Ġrec urrent +Ġdev ils +Ġwater fall +æĺ ¯ +go o +Text Color +Ġtam pering +IV ES +TR UMP +ĠAb el +ĠS AL +ĠHend ricks +ĠLu cius +b ots +Ġ40 96 +IST ORY +Gu est +ĠN X +in ant +Ben z +ĠLoad ed +ĠCle ver +t reatment +Ġta vern +Ġ3 39 +ĠT NT +ific antly +Tem perature +F el +Ġunder world +ĠJud ges +Ġ< + +Ġst ump +Ġoccup ancy +Ġab er +ĠF inder +) ", +ĠN unes +res et +in et +ect omy +Ġwell ness +ĠP eb +quart ered +and an +Ġneg atives +ĠTh iel +ĠCl ip +ĠL TD +Ġbl ight +Ġreperto ire +K yle +Ġqu er +ĠC es +Ġha pl +98 9 +ĠTh ames +isc opal +Des k +ivari ate +ĠEx cellence +found ation +Ġâ ĩ +X i +Ġmyster iously +esty les +Ġper ish +ĠEng els +ĠDE AD +09 0 +}} } +ĠUn real +Ġrest less +ID ES +orth odox +ĠInter mediate +Ġdin ners +ĠTr out +ĠSe ym +ĠHall s +og ged +Ġtraged ies +Ġdid nt +67 6 +Ġail ments +Ġobserv able +ĠV ide +ad apt +ĠD usk +Ġprofessional ism +ĠPres cott +ĠInd ies +p ox +ĠMe hran +W ide +Ġend emic +ĠPar an +B ird +Ġped als +ĠI U +ĠAdam ant +ĠH urt +Ġcorrel ates +urd en +Ġspons oring +cl imate +ĠUnivers ities +ĠK not +enn es +ĠDam ian +ĠAx el +S port +Ġbar b +ĠS no +sh own +ste en +ud ence +Ġnon violent +Ġhom ophobia +Ġbiom ass +ĠDet ail +Ġsrf N +ĠT une +accompan ied +I ENCE +Al bert +ĠMong o +z x +ĠCer berus +or bit +c ens +Ġsl ay +SH ARE +H Y +Ġb rawl +ĠPro be +Ġnonex istent +ĠClare nce +ĠBlack burn +Ġport als +ĠR ita +ĠRem ain +ĠLe vant +Ġtrick ed +ĠF erry +aver ing +ĠStraw berry +ĠAn swers +Ġhorrend ous +ĠA man +Supp lement +ĠT oad +Ġpe eled +Ġman oeuv +ĠU zbek +mond s +ĠH ector +Ġ40 2 +pe es +fix es +Ġd j +Ġres umes +Ġaccount ant +Ġadvers ity +Ġham pered +ĠL arson +Ġd oping +part s +H ur +Ġbe arded +Ġy r +ĠPlug in +å¥ ³ +Ġ/ ** +rol ley +Ġwaters hed +ĠSub mission +if lower +AS C +Ġcho ir +Ġsculpt ures +m A +incre asing +ai i +Ġsne akers +Ġconfront s +ĠEle phant +ĠEl ixir +Ġrec al +ĠT TL +w idget +ĠW ax +ĠGr ayson +Ġha irst +Ġhumili ated +ĠWAR N +app iness +ĠT TC +F uel +Ġpol io +Ġcomplex es +Ġbab e +ĠX IV +P F +). [ +P arts +Ġ4 35 +M eg +ĠY ards +ĠAL P +Ġy ells +Ġprin ces +Ġbull ies +ĠCapital ism +ex empt +FA Q +ĠSp onge +ĠAl a +Ġpleas antly +Ġbu f +Ġden ote +Ġunp ublished +Ġkne eling +asc a +Ġl apse +al ien +99 4 +Ġrefere es +ĠLaw yers +S anta +Ġpuzz ling +ĠProm etheus +ĠPh araoh +ĠDel ay +Ġfacilit ates +ĠC ES +Ġjew els +Ġbook let +ond ing +Ġpolar ization +ĠMor an +ĠSal ad +ĠS OS +ĠAdv ice +PH OTOS +IC AN +iat ures +ex press +ĠWonder land +ĠC ODE +ĠCL ASS +9 75 +Ġg rep +ĠD iesel +ĠGl ac +! ?" +Ġr m +o ine +disc rimination +ĠN urse +m allow +Ġv ortex +ĠCons ortium +Ġlarge Download +stra ight +augh lin +G rad +Ġpublic ized +ĠW aves +ĠRed d +Ġfest ivities +ĠM ane +ar ov +Ġfleet ing +ĠDr unk +ug en +C ele +Ġchromos omes +ĠD OT +-+-+ -+-+ +Ġbus iest +ĠBe aver +Sy rian +ĠK yr +k as +ĠCross Ref +19 50 +76 01 +Ġrepe aling +ĠWin ners +ĠMac ro +ĠD OD +bl ance +S ort +64 1 +Ġmet re +ĠD irk +Ġgo ggles +Ġdraw backs +Ġcomplain ant +Ġauthor izing +Ġantit rust +oper ated +Ġm ah +Ġexagger ation +Am azing +ĠSer aph +Ġha ze +w ow +Ġextingu ished +Ġcan yon +ĠB osh +Ġv ents +Ġsc rape +Cor rect +4 26 +Ġav g +Dem and +ĠâĪ ¼ +Ġmicrobi ota +"} ]," +ĠSt ev +B io +ĠPlan es +Ġsuggest ive +Ġdec ipher +ĠRefuge e +ĠKe jriwal +ĠGreen peace +Ġdecl ass +ĠSound ers +Ġth o +Ġdec rypt +Ġbr ushing +ĠJane iro +ip op +S i +8 77 +ĠGeoff rey +Ġc pu +ĠHaz el +Ġview points +Ġcris py +ĠNot ification +Ġsold er +ĠMod est +ĠHem isphere +Ġcass ette +in cludes +Ġident ifiers +ĠC ALL +in cent +T odd +ĠSwe ep +Ġ3 34 +b oss +Ġsm ir +gin x +Ġtown ship +Ġg rieving +ĠMos que +Net flix +AS ED +ĠMillenn ials +oc om +19 67 +Ġbold ly +s leep +Ġes che +arij uana +Ġsw irl +ĠPen al +Ġneglig ent +ĠStephen son +K ER +ĠZ oro +ris is +Ġlocal ization +ĠSeym our +ĠAng lic +red itation +prot ection +ĠPa ige +Ġo mit +ĠR ousse +ĠT ub +Ġinv itations +t ty +Ġm oss +ph ysical +C redits +Ġan archy +Ġchild care +Ġl ull +ĠM ek +ĠL anguages +lat est +ĠSan ford +Ġus ability +Ġdiff use +ĠD ATA +Ġsp rites +ĠVeget a +ĠProm otion +ãĥ¼ ãĤ¯ +rict ing +z ee +Tur kish +ĠTD s +pro ven +57 1 +Ġsmug glers +707 10 +Ġreform ed +ĠLo is +Ġun fl +ĠWITH OUT +ĠReturn ing +ann ie +ĠTom as +Fr anc +ĠProf it +ĠSER V +ĠR umble +ik uman +es an +Ġt esters +Ġgad get +Ġbrace let +ĠF SA +comp onent +Ġparamed ics +Ġj an +ĠRem em +ĠSk inner +Ġl ov +ĠQu ake +rom a +Ġfl ask +Pr inc +Ġover power +Ġlod ging +ĠK KK +ret te +Ġabsor bs +w rote +Ġ ," +K ings +ĠH ail +ĠFall ing +xt ap +ĠHel ena +ire ns +L arry +Ġpamph let +ĠC PR +G ro +ĠHirosh ima +Ġhol istic +". [ +Ġdet achment +Ġas pire +Ġcompl icit +ĠGreen wood +Ġresp awn +ĠSt upid +ĠFin ished +f al +b ass +Ġab hor +Ġmock ery +ĠFe ast +VID EO +Ġcon sec +ĠHung ry +P ull +ĠH ust +it ance +? ãĢį +) -- +ĠPar allel +con v +4 69 +ha ar +w ant +P aper +m ins +ĠTor o +ĠTR UMP +ĠR ai +D W +ĠW icked +ĠL ep +Ġfun ky +Ġdetrim ent +ios is +ache v +Ġde grade +im ilation +Ġret ard +Ġfrag mentation +Ġcow boy +ĠY PG +ĠH AL +Parent s +ĠS ieg +ĠStra uss +ĠRub ber +× IJ +Fr ag +Ġp t +Ġoption ally +ĠZ IP +ĠTrans cript +ĠD well +88 2 +M erc +ĠM OT +ãĥ¯ ãĥ³ +Ġhun ts +Ġexec utes +In cludes +Ġacid ic +ĠRespons ibility +ĠD umb +we i +And erson +ĠJas per +ight on +abs olutely +Ad ult +Ġpl under +Mor ning +ĠT ours +ĠD ane +Î º +ĠT EST +ĠG ina +Ġcan ine +aw an +Ġsocial ists +ĠS oda +Ġimp etus +ĠSupplement ary +oli ath +ĠKinn ikuman +mitted ly +second s +Ġorganis ers +Ġdocument aries +Vari able +GRE EN +Ġres orts +Ġbr agging +Ġ3 68 +Art ist +w k +bl ers +Un common +ĠRet rieved +Ġhect ares +Ġtox in +r ank +Ġfaith s +ĠG raphic +Ġve c +ĠL IA +Af rican +Ġard ent +end iary +L ake +ĠD OS +cient ious +ĠOk awaru +ĠAll y +ĠTim eline +D ash +ĠI c +contin ue +Ġt idy +Ġinstinct ively +ĠP ossibly +ĠOut door +ĠWould n +Ġl ich +ĠBr ay +ĠA X +Ġà ī +Ġ+ # +\ ' +Direct ory +ab iding +Ġf eral +ic ative +but t +Ġper verse +S alt +Ġwar ped +Ġnin eteen +Ġcabin ets +Ġsrf Attach +ĠSl oan +Ġpower ing +reg ation +F light +se vere +Ġst ren +Ġc og +ap ache +Ġâ Ŀ +Ġcaf eteria +p aces +ĠGrim oire +uton ium +Ġr aining +Ġcir cling +Ġlineback ers +c redit +Ġrep atri +ĠCam den +lic ense +Ġly ric +Ġdescript or +Ġval leys +Ġre q +Ġback stage +ĠPro hibition +ĠK et +Op ening +S ym +æĸ ¹ +Ġserv ings +Ġoverse en +Ġaster oids +ĠMod s +ĠSpr inger +ĠCont ainer +è » +ĠM ens +Ġmult im +Ġfire fighter +pe c +Ġchlor ine +Ð ¼ +end i +Ġsp aring +Ġpolyg amy +ĠR N +ĠP ell +Ġt igers +Ġflash y +ĠMad ame +S word +Ġpref rontal +Ġpre requisite +uc a +Ġw ifi +Ġmiscon ception +Ġharsh ly +ĠStream ing +ot om +ĠGiul iani +foot ed +Ġtub ing +ind ividual +z ek +n uclear +m ol +Ġright ful +49 3 +Ġspecial ization +Ġpassion ately +ĠVel ocity +ĠAv ailability +T enn +Ġl atch +ĠSome body +Ġhel ium +cl aw +Ġdi pping +XX X +Ġinter personal +7 10 +Ġsub ter +Ġbi ologists +ĠLight ing +Ġopt ic +Ġden im +end on +ĠC orm +Ġ3 41 +ĠC oup +Ġfear less +Ġal ot +ĠCliff ord +ĠRun time +ĠProv ision +up dated +lene ck +Ġneur on +Ġgrad ing +ĠC t +sequ ence +in ia +con cept +Ġro aring +ri val +ĠCaucas ian +Ġmon og +key es +Ġappell ate +Ġlia ison +EStream Frame +ĠPl um +! . +Ġsp herical +Ġper ished +Ġbl ot +Ġben ches +Ġ4 11 +Ġpione ered +Ġhur led +Jenn ifer +ĠYose mite +Ch air +Ġreef s +Ġelect or +ĠAnt hem +65 2 +Ġun install +Ġimp ede +Ġbl inking +Ġgot o +Dec re +A ren +Ġstabil ization +ĠDis abled +ĠYanuk ovych +Ġoutlaw ed +ĠVent ura +ten ess +Ġplant ation +Ġy acht +ĠHu awei +Ġsol vent +Ġgr acious +Ġcur iously +Ġcapac itor +Ġc x +ĠRef lex +Ph ys +ĠC f +pt in +cons ervative +Ġinv ocation +c our +F N +ĠNew ly +H our +As ian +ĠLe ading +ĠAer ospace +An ne +Ġpre natal +Ġdeterior ating +H CR +ĠNorm andy +ol ini +ĠAm bro +9 10 +Ġset backs +ĠT RE +Ġs ig +ĠSc ourge +59 7 +79 8 +Game play +Ġm sec +M X +Ġprice y +ĠL LP +aker u +Ġover arching +ĠB ale +Ġworld ly +Cl ark +Ġscen ic +Ġdisl iked +ĠCont rolled +T ickets +ĠE W +ab ies +ĠPl enty +Non etheless +Ġart isan +Trans fer +ĠF amous +Ġinf ield +ble y +Ġunres olved +ĠML A +ãĤ Ĥ +Cor rection +Ġdemocr at +ĠMore no +ro cal +il ings +Ġsail or +Ġr ife +h ung +Ġtrop es +Ġsn atched +ĠL IN +ĠB ib +ES A +ĠPre v +ĠCam el +run time +Ġob noxious +4 37 +Ġsum mers +Ġunexpl ained +ĠWal ters +cal iber +Ġg ull +ĠEnd urance +ä½ ľ +Ġ3 47 +Ir ish +Ġaer obic +Ġcr amped +ĠHon olulu +à © +us erc +ec ast +AC Y +ĠQu ery +ãĤ¹ ãĥĪ +Bet a +Ġsuscept ibility +ĠSh iv +ĠLim baugh +Ġà ĸ +ĠN XT +ĠM uss +ĠBrit ons +ES CO +EG IN +Ġ% % +Ġsec ession +ĠPat ron +ĠLu a +n aires +ĠJPM organ +us b +ocy te +Ġcouncill ors +ĠLi ang +f arm +Ġnerv ously +Ġattract iveness +ĠK ov +j ump +Pl ot +Ġst ains +ĠStat ue +ĠApost les +he ter +ĠSUP PORT +Ġoverwhel m +Y ES +Ġ29 1 +d ensity +Ġtra pping +M it +Ġf ide +ĠPam ela +atl antic +Dam n +Ġp ts +OP A +Ġserv icing +Ġoverfl owing +ul o +ĠE rit +t icket +light ing +ĠH mm +ãĥ¼ ãĥ« +im oto +Ġchuck le +4 23 +ãģ ķ +sh ape +Ġque ues +Ġanch ors +ãĤ¼ ãĤ¦ãĤ¹ +F er +Ġaw oke +Ġ6 66 +h ands +Ġdiver gence +Ġ50 5 +T ips +Ġdep ot +Ġske w +ĠDel iver +op ot +Ġdiv ul +ĠE B +uns igned +ĠUn i +X box +Ġfor ks +Ġ7 02 +å ¯ +Ġpromot ers +ĠV apor +Ġlev ied +sl ot +Ġpig ment +Ġcyl inders +C RE +Ġsn atch +Ġperpet ually +Ġl icking +ĠFe et +ĠKra ken +ĠHold en +ĠCLS ID +m r +Ġproject or +Ġden otes +Ġchap el +ĠTor rent +b ler +R oute +ĠDef endant +ĠPublisher s +ĠM ales +ĠInn ov +ĠAg ility +rit er +ty mology +st ores +L ind +Ġf olly +ĠZur ich +B le +Ġnurt ure +Ġcoast line +uch in +D omin +Ġfri vol +ĠCons olid +res ults +M J +Ġphyl ogen +Ġha uled +ĠW iley +ĠJess ie +ĠPrep are +ĠE ps +Ġtreasure r +I AS +Ġcolon ists +Ġin und +ĠWW F +ĠCon verted +6 000 +out side +ĠApp earance +ĠRel ic +ĠM ister +s aw +Ġresult ant +Ġadject ive +ĠLaure l +ĠHind i +b da +Pe ace +Ġreb irth +Ġmembr anes +Ġforward ing +Ġcoll ided +ĠCar olyn +K ansas +5 99 +ĠSolid GoldMagikarp +Be ck +Ġstress ing +ĠGo o +ĠCooper ative +Ġf s +ĠAr chie +L iter +ĠK lopp +J erry +Ġfoot wear +War ren +Ġsc ree +h are +Under standing +P ed +Ġanth ology +ĠAnn ounce +M ega +Ġflu ent +Ġbond age +ĠDisc ount +il ial +C art +ĠNight mares +Sh am +ĠB oll +uss ie +H ttp +Atl anta +Ġun recogn +ĠB id +Ġunder grad +Ġforg iving +ĠGl over +AAAA AAAA +4 45 +V G +pa io +kill ers +Ġrespons ibly +Ġmobil ize +Ġeffect ed +ĠL umin +Ġk ale +Ġinfring ing +ann ounced +Ġf itt +b atch +ĠT ackle +ĠL ime +ĠAP P +uke mia +Ġrub y +Ġex oner +ĠCas ual +0 70 +Ġpel vic +Ġautom ate +ĠK ear +ĠCoast al +Ġcre ed +Ġbored om +ĠSt un +ri ott +Ĥ İ +Ġregener ate +Ġcomed ians +ĠOP ER +Sp ons +id ium +on is +L ocated +05 7 +Ġsusp ense +ĠD ating +C ass +Ġneoc ons +ĠShin zo +Ġaw oken +ch rist +ĠMess ages +att led +ĠSpr ay +ĠSp ice +C W +Ġshield ing +ĠG aul +Am id +Ġparam ilitary +Ġmult if +ĠTan ner +il k +Ġgodd amn +g ements +Ġbe friend +m obi +Ġ3 88 +fold er +acc a +Ġins in +g ap +N ev +fif th +Ġpsychiat ry +b anks +TH IS +Ġhar b +ac qu +Ġfac ade +ĠPower Point +80 3 +Ġbl uff +Sh ares +Ġfavor ing +El izabeth +Ãį Ãį +Ġr anger +77 2 +ĠAr che +h ak +ĠGen etics +ĠF EMA +Ġev olves +Ġest e +ĠP ets +ĠM é +ĠInterest ing +ĠCanter bury +ch apter +ĠStar fleet +Sp anish +Ġdraw back +ĠNor wich +9 70 +n orth +ag anda +Ġtransform ative +ram ids +bi ology +ad ay +Ġpropag ation +ĠGam ma +ĠDen ise +ĠCalcul ator +ent imes +ĠB ett +Ġapp endix +ĠHD D +AK ING +Ġst igmat +Ġhol ster +Ġord inarily +Ch ance +ĠCont rary +Ġad hesive +Ġgather s +6 12 +re au +ony ms +ew ays +Ġindu ces +Ġinterchange able +se m +Wh it +Ġtr ance +Ġincorpor ation +ĠExt ras +Fin ancial +Ġawkward ly +ĠStur geon +ĠH Y +Norm ally +ĠEnd ing +ĠAss ist +enc rypted +Ġsub jug +Ġn os +Ġfan atic +C ub +C U +?" . +Ġirre versible +å Ĥ +03 1 +ĠH AR +sp read +ul ia += $ +Sc ope +L ots +Ġlif estyles +ol on +Ġf eds +Ġcongrat ulate +web kit +Ġindist inguishable +ĠSw ing +Ġcommand ments +qu ila +ab ella +m ethyl +ann abin +Ġo vere +Ġlob ster +ĠQU EST +ĠCONT IN +bern atorial +:::: :::: +ĠTra ve +ĠSam oa +AN I +75 2 +Ð ´ +userc ontent +ĠMod erate +y eah +ĠK itt +Ġwe e +Ġstuff ing +ĠInter vention +ĠD ign +Ġware houses +ĠF iji +Ġpel lets +Ġtake away +ĠT ABLE +ĠClass ical +col lection +Ġland fall +ĠMus cle +Ġsett les +ĠAD V +Ġ3 44 +L aura +Ġf ared +ĠPart ial +4 36 +oss ibility +ĠD aly +ĠT arant +ĠFu ji +am l +c ence +55 1 +ĠProced ures +ĠO CD +ĠU D +t in +Q UI +ach o +4 38 +Ġgl itches +Ġenchant ment +Ġcalcul ates +IR O +ĠH ua +alys es +ĠL ift +um o +Ġle apt +Ġhypothes ized +ĠGust av +it ans +VERS ION +æ ł +Rog er +Ġr and +ĠAd apter +Ġ3 31 +ĠPet ition +k ies +M ars +Ġunder cut +ze es +ĠLy ons +ĠDH CP +Miss ing +Ġretire es +Ġins idious +el i +> ) +. ãĢį +Ġfinal ists +ĠA ure +Ġacc user +Ġwas tes +ĠY s +ĠL ori +Ġconstitu encies +Ġsupp er +Ġmay hem +or ange +Ġmis placed +Ġmanager ial +Ġex ce +ĠCL I +Ġprim al +ĠL ent +Cry stal +h over +ĠN TS +end um +Ġd w +ĠAl c +n ostic +Ġpres erves +ĠTs arnaev +Ġtri pled +rel ative +Arc ade +k illing +ĠW EEK +ĠH anna +D ust +Com pleted +ģ « +Ġappro ves +ĠSur f +ĠLuther an +ven ants +Ġrobber ies +we ights +soft ware +at ana +ug al +Ġgrav y +ĠC ance +OLOG Y +ly ak +Ton ight +Ġunve il +Ġ19 04 +ĠMin ion +ent ious +st ice +pack ages +ĠG EAR +Ġg ol +ĠHutch inson +ĠProf ession +ĠG UN +ĠDiff erence +ĠTsuk uyomi +ĠLes bian +6 70 +Ġfug itive +ĠPlan etary +-------------------------------- ------------------------ +Ġacc rued +Ġch icks +Ġsto pp +Ġblock ers +C od +Ġcomment ers +ĠSomew here +ĠPhot ographer +the me +Ġmay oral +w u +Ġanten nas +Ġrev amped +ĠSubject s +it é +im ura +Ġentr ances +liter ally +Ġten ets +ĠO MG +ĠMP H +ĠDon key +ĠOff ense +Ġ" + +Sn ap +ĠAF B +Ġan imate +ĠS od +His panic +Ġinconsist ency +D b +F Y +Ex port +Ġa pe +Ġpear l +ib el +ĠPAC s +Ġ{ \ +Ġact u +ĠHS BC +camp us +Ġpay off +Ġde ities +ĠN ato +ou ple +Ġcens ored +ĠCl ojure +Ġconf ounding +en i +Ġreck on +op he +Ġspot ting +Ġsign ifies +Ġprop el +Ġfest ive +S uggest +Ġpled ging +ĠB erman +Ġrebell ious +Ġovershadow ed +Ġinfiltr ated +j obs +67 2 +Ġscal able +Ġdomin ion +ĠNew foundland +ĠMead ow +Ġpart itions +AM I +Ġsupplement ary +str ument +Ġhair y +Ġperpet uate +Ġnuts hell +ĠPot ato +ĠHob bit +Ġcur ses +Flo at +Ġquiet er +Ġfuel ing +Ġcaps ules +ĠL ust +ĠH aunted +Exec utive +Ġchild birth +G re +Ġrad iant +å İ +Ġm alls +Ġin ept +ĠWarrant y +Ġspect ator +E h +t hens +Ġculmin ating +æ © +ary a +ãĤ ® +ilit arian +ĠOR IG +ĠSp ending +pt ives +ĠS iren +ĠRec ording +ay ne +Ġv im +Ġspr ang +T ang +ĠM FT +mor ning +ĠWe ed +m peg +cess ion +ĠCh ung +7 30 +w arning +56 2 +handed ly +P oor +P olitics +: # +Ġp ian +Ġfec es +ĠDocument ation +Ġban ished +Ġ3 99 +ĠAR C +Ġhe inous +J ake +ĠAm ir +way ne +v re +os henko +Ġnotebook s +Ġfound ational +Ġmarvel ous +ixt ape +Ġwithdraw als +Ġh orde +ĠD habi +is able +ĠK D +Ġcontag ious +ĠD ip +ĠAr rows +Ġpronoun s +Ġmorph ine +ĠB US +68 2 +Ġk osher +fin ished +ĠInstr uments +Ġf used +yd en +ĠSal mon +F ab +aff ected +K EN +C ENT +Dom ain +Ġpoke mon +ĠDr inking +G rowing +ĠInvestig ative +ĠA ether +em i +Ġtabl oid +Ġrep ro +ĠNot withstanding +ĠBers erker +Ġdram as +Ġclich é +Ġb ung +ĠU RI +ĠD os +0 44 +Ġpast ors +Ġl s +Ġac rylic +aun ts +Ed ward +Ġmajor ities +B ang +Ġfield ing +ĠRepl acement +ĠAl chemy +pp ard +ĠRome o +ĠSan ct +ĠLav rov +ib ble +Inst ruct +Ġimp ractical +ĠPlay boy +ce phal +Ġsw aps +Ġk an +ĠThe o +Ġillust rating +Ġdismant led +ĠTrans gender +ĠG uth +UG H +Ġtriumph ant +Ġencomp ass +Ġbook mark +udd in +j er +Ġpred icate +ES H +Ġwhen ce +ĠAB E +Ġnon profits +Se qu +Ġdi abetic +Ġp end +Ġheart felt +sh i +Ġinter acts +ĠTele com +Ġbombard ment +dep ending +ĠLow ry +ĠAd mission +ĠBl ooming +ust ration +ene gger +B rew +Ġmol ten +ĠNer d +P IN +âĸ Ģ +ave ment +Ġtou red +Ġco efficients +ĠTray von +ans son +Ġsand y +t old +fl ows +Ġpop ulous +ĠT inder +ĠBl iss +R achel +Min imum +Ġcontest ant +ĠRed uce +ĠMor se +ĠGrass ley +ĠClick er +Ġexp r +Ġs incerity +Ġmar qu +Ġelic it +ĠPro position +ĠDemon ic +Ġtac os +G reek +Ġpost war +Ġin sofar +ĠP ork +Ġ35 2 +doctor al +walk ing +Ġmid term +ĠSam my +sight ed +ĠTR ANS +ic i +AL D +ĠUS L +ĠF ISA +ĠAm pl +ĠAlex andra +ine lli +Tr ain +Ġsign ify +ĠVers us +Ġob fusc +Ġk h +Ġagg ro +ĠRen ault +Ġ3 48 +5 18 +ox icity +0 22 +ĠTw ist +Ġgoof y +D ynamic +Ġbrief ings +m ight +8 99 +Ġderog atory +T ro +Ġfor ging +ĠKor an +ĠMar ried +ĠBuc s +Ġpal ate +ĠCon version +m able +4 13 +Ġ( _ +Ġs iph +ĠN EO +col lege +Ġmarg inally +Ġfl irt +ĠTra ps +ĠP ace +é »Ĵ +Ġgoalt ender +Ġforb ids +Ġcler ks +ĠT ant +ĠRobb ins +ĠPrint ing +Ġpremie red +Ġmagn ification +ĠT G +ĠR ouse +ĠM ock +odynam ics +Ġpre clude +ism o +ĠPul itzer +Ġaval anche +ĠK odi +rib une +ĠL ena +Elect ric +Ġref inery +Ġend owed +Ġcounsel ors +Ġd olphin +ĠM ith +Ġarm oured +hib ited +Beg in +ĠP W +O il +ĠV or +ĠShar if +ĠFraz ier +est ate +Ġj ams +Pro xy +Ġband its +ĠPresbyter ian +ĠPrem iere +t iny +ĠCru el +Test ing +Ġhom er +ĠV ERS +ĠPro l +ĠDep osit +ĠCoff in +Ġsemin ars +Ġs ql +ĠDef endants +Altern atively +ĠR ats +ç « +ethy st +' > +Ġiss uer +58 9 +Ġch aired +ĠAccess ories +man ent +Ġmar row +ĠPrim ordial +C N +Ġlimit less +ĠCarn age +Ġund rafted +q v +IN ESS +on ew +Ġco hesion +98 7 +Ġne cks +Ġfootball er +ĠG ER +Ġdetect able +ĠSupport ing +ĠCS V +oc ally +k Hz +Ġund e +Ġsh one +Ġbud ding +tra k +Stand ing +ĠStar craft +ĠKem p +Ben ch +Ġthw arted +ĠGround s +ath i +L isa +Dial og +ĠS X +V ision +Ġingen ious +Ù IJ +Ġfost ering +ĠZ a +ĠIn gram +Ġ" @ +N aturally +6 16 +0 35 +ĠF AC +H mm +55 4 +Ġacceler ator +ĠV end +Ġsun screen +Ġtuber culosis +rav iolet +ĠFunction al +ĠEr rors +ed ar +19 66 +ĠSpect re +ĠRec ipes +88 5 +ĠM ankind +L iverpool +Ġ| -- +Ġsubst itutes +ĠX T +w ired +Ġinc o +ĠAf gh +E va +ic c +S ong +K night +Ġdilig ently +ĠBroad cast +A id +Ġaf ar +ĠH MS +aton in +ĠGr ateful +Ġfire place +ĠOm ni +e uro +ĠF RE +ĠSh ib +ĠDig est +t oggle +Ġheads ets +Ġdiff usion +ĠSqu irrel +ĠF N +Ġdark ened +out her +Ġsleep s +ĠX er +gun s +Ġset ups +Ġpars ed +Ġmamm oth +ĠCur ious +g ob +ĠFitz patrick +ĠEm il +im ov +........ ..... +ĠB enny +Second ly +Ġheart y +Ġcons on +st ained +Ġgal actic +cl ave +Ġplummet ed +Ġp ests +Ġsw at +Ġrefer rals +ĠLion el +h oly +Ġunder dog +ĠSl ater +ĠProv ide +ĠAm ar +ress or +å Į +ong a +Ġtim id +Ġp iety +ĠD ek +Ġsur ging +az o +Ġ6 10 +Ġdes ks +ĠSp okane +ĠAn field +Ġwars hips +ĠCob ra +Ġar ming +clus ively +ĠBad ge +ag ascar +ĠPR ESS +ĠMcK enzie +ĠFer dinand +burn ing +Af ee +Ġtyr ann +ĠI w +ĠBo one +100 7 +ĠRe pt +Ċ Âł +Ġcar avan +ĠD ill +ĠBundes liga +Ch uck +Ġheal er +ãĥ¼ãĥ Ĩ +ĠH obby +Ġneg ate +Ġcrit iques +section al +mop olitan +Ġd x +Ġouts ourcing +ĠC ipher +t ap +Sh arp +Ġup beat +Ġhang ar +Ġcru ising +ĠNi agara +Ġ3 42 +ill us +ĠS v +Ġsubt itles +Ġsqu ared +Ġbook store +Ġrevolution aries +ĠCarl ton +ab al +Ut ah +Ġdesp ise +ĠU M +cons ider +aid o +Ġc arts +ĠT urtles +Tr aining +Ġhonor ary + ¢ +Ġtri angles +4 22 +Ġreprint ed +Ġgrace ful +ĠMong olia +Ġdisrupt ions +ĠB oh +Ġ3 49 +Ġdr ains +Ġcons ulate +Ġb ends +Ġm afia +ur on +ĠF ulton +m isc +Ġren al +Ġin action +ck ing +Ġphot ons +Ġbru ised +ĠC odes +og i +Ġn ests +ĠLove ly +ĠLib re +ĠD aryl +Ġ# ## +S ys +. ," +Ġfree zes +est ablishment +and owski +Ġcum bers +ĠSt arg +ĠBom bs +Ġleg ions +Ġhand writing +Ġgr un +ĠC ah +sequ ent +Ġm oth +ĠMS M +Ins ert +F if +Ġmot el +Ġdex ter +ĠB ild +hearted ly +Ġpro pe +ĠText ure +ĠJ unction +ynt hesis +oc ard +ĠVer a +ĠBar th +Ġμ g +Ġl ashed +Ġ35 1 +ĠZ amb +ĠSt aples +ĠCort ex +ĠCork er +Ġcontinu um +ĠWR ITE +unt a +rid or +Ġde ems +0 33 +ĠG OLD +p as +Ġrep ressive +ãĥĨ ãĤ£ +Ġbaff led +Sc ar +Ġc rave +Ġ ______ +Ġentrepreneurs hip +ĠDirector ate +Ġ' [ +Ġv ines +Ġasc ended +ĠGR OUP +ĠGood bye +Ġdo gged +ãĥ´ ãĤ¡ +Man ufact +Ġunimagin able +ri ots +ier rez +Ġrel ativity +ĠCraft ing +ra ught +ud en +c ookie +Ġassass ins +Ġdissatisf ied +ac ci +Ġcondu it +Sp read +ĠR ican +n ice +izz le +Ġsc ares +ĠWH Y +ph ans +5 35 +Ġprot racted +ĠKrist en +5 36 +ĠSc rib +ĠNe h +Ġtwent ies +Ġpredic ament +Ġhandc uffs +Ġfruit ful +ĠU L +ĠLud wig +Ġatt est +ĠBre aker +Ġbi ologically +ĠDeal er +Ġrenov ations +f w +ess en +Al ice +ĠHen ri +Ġun ilaterally +ĠS idd +h ai +ĠSt retch +S ales +Ġcumbers ome +ĠJ avier +Ġtrend y +Ġrot ting +ĠChall enges +Ġscra ps +Ġfac ets +ĠVer onica +ĠVer ge +ĠS ana +Al ien +ĠR ih +Ġrad ial +ect ar +Ġ6 30 +cl i +Mar ie +Ġwild fire +ĠCat o +h ander +Ġwait ress +Ġch ops +ĠS ECTION +Ġblunt ly +ĠCat alog +n ian +stud y +Ġpat rolling +ĠT enth +nex us +ĠN ON +op sy +Ġsc athing +s ie +Ġdeterior ated +V B +Naz is +Ġdep ictions +Ġauthent icated +ĠCon ce +k rit +Ġpromul g +ĠL ONG +U FC +ĠVis itors +ĠRec all +Ġrehab ilit +ĠSL I +Ġglac ier +ĠB ite +Ġ50 3 +Ġvom it +Ġfer mented +ĠKh alid +Ġgrad ed +ĠMag icka +ĠIch igo +power ful +ic ators +75 3 +Ġsh rew +Ġ35 6 +Ġlegal izing +Ġall otted +ĠArch demon +ith ing +igg urat +V OL +Le od +Ġo ily +Ġindu cing +Ġamy gdala +Ġadm ins +ĠAcqu isition +C AN +Ġsche matic +Ġmo an +ĠCamer oon +Ġt ink +Ġmer ry +Ġbutter flies +ĠGo ff +Ġworks pace +ĠCor ona +Ġj avascript +ĠD olphin +ĠCant or +4 64 +to e +AP S +ĠAg ing +Ġpadd ed +ĠZ heng +ĠHe ld +Ġest ranged +Ġ7 70 +. } +ĠDun ham +Ġsm okes +Ġcap itals +und ai +Sh in +ĠFound ing +Ġent itle +Ġcenter piece +D iscover +Ġthere to +al ert +ĠN ou +ĠAnaly st +l c +F H +FI ELD +ĠP OV +gr ay +Ġar cs +ĠH OT +Ġr s +Ġoblig atory +ĠArchitect s +ĠS ven +ĠF EC +0 200 +Christ mas +ĠAlban ia +rat om +58 7 +Ġhard ships +Ġaut os +ĠCharg es +Ġap es +Ġ3 76 +wal let +Ġintox ication +Ġgobl in +Ġ5 70 +++++++++ ++++++++ +ĠYel p +ĠMag netic +ĠBr iggs +R ail +Ġspawn s +ĠW iggins +Ġshowc ased +Ġres orted +ub en +Ġwh ipping +Ġim itate +Ġdigest ion +ĠUS PS +ĠG est +Ġye a +ĠT ight +ind al +ic as +` . +C AST +'' ; +ĠF et +opath ic +In valid +Ġregrett ed +Ġbro ccoli +ĠSc ores +e ve +Ġpost ings +Ġaccum ulating +Ġneed less +elf th +Ġmay ors +Ġsc rib +Ġanecd otes +Ġbot ched +ĠRib bon +ĠConstant ine +i uses +ess es +Ġdev ise +Comp ared +Ġp udding +Ġg arg +Ġev oke +79 7 +Ġdet ox +9 09 +ĠPie ces +ĠMcC artney +Ġmet ast +ĠK rypt +P OR +Ġt ending +ĠMerch ants +Pro of +ĠV arg +ĠPort able +ãĥ¼ãĥĨ ãĤ£ +B rain +25 00 +Ġfol iage +Ø ¹ +Ġment ors +ĠA ires +Ġminimal ist +Ġing ested +ĠTro jan +ĠQ ian +inv olved +0 27 +Ġer oded +RA FT +Ġbl urry +M ob +Ġbuff et +ĠFn atic +ae a +KN OWN +ĠIn it +s afety +en um +ACT ION +ĠCrus her +ĠD ates +Ġ ................ +c alling +ak ov +Ġvent ured +Ġ5 55 +au ga +H art +ĠA ero +M AC +Ġthin ly +Ġar ra +ST ATE +ild e +ĠJac qu +ĠFem ales +Ġthe orem +Ġ3 46 +Ġsmart est +ĠPU BLIC +ĠK ron +ĠB its +ĠV essel +ĠTele phone +Ġdec ap +Ġadj unct +ĠS EN +mer ga +Ġred acted +Ġpre historic +Ġexplan atory +ĠRun s +ĠUtt ar +ĠM anny +ĠAUTH OR +ĠUnle ashed +ĠBow ling +be ans +79 3 +Ġunivers es +Ġsens it +ĠK ung +re peat +ctr l +Ġp aced +Ġfull er +Cl ock +Ġrec omb +ĠF aul +ĠB unker +Ġpool ed +Ġan a +ĠM outh +LL OW +hum ane +Ġbull do +ĠMicha els +f am +Ġwreck ed +Ġport rays +ĠWh ale +ĠH es +Ġguess es +ĠBrow se +ĠL APD +Ġconsequ ential +ĠInn ocent +ĠD RAG +Ġtrans gress +ĠO aks +Ġtri via +ĠRes on +ĠA DS +-- + +ĠT oll +Ġgrasp ing +ĠTHE M +ĠT ags +ĠCon clusion +Ġpract icable +Ġho op +Ġunintention ally +Ġign ite +ĠM ov +ur ized +le hem +Ter min +Ġcolour ful +ĠLin ear +ĠEll ie +G y +Ġman power +Ġj s +Ġem oji +ĠSHAR ES +_ . +0000 7 +Ġsophistic ation +Ġunders core +Ġpract ise +Ġbl ob +op ens +Uk raine +Ke eping +Y C +J R +ult imate +Cl aim +Ġautom obiles +99 3 +ste el +Ġpart ing +ĠL ank +... ? +Ġ38 5 +Ġremem brance +Ġe ased +Ġcov ari +ĠS ind +Effect ive +Ġdisse mination +ĠMo ose +ĠCl apper +br ates +App ly +Ġinv is +Ġwors ened +âĢĶ - +Ġlegisl ator +ĠL ol +ĠRow e +Ġdealers hip +um ar +id ences +Ġinvestig ates +Ġc ascade +Ġbid der +ĠB EN +Iron ically +Ġpres iding +Ġd ing +Ġcontrad icted +Ġshut s +ĠF IX +Ġ3 66 +Dist rict +Ġsin ful +ĠChar isma +o ops +Ġtot ality +Ġrest itution +ĠOpt imus +ĠD ah +Ġcl ueless +urn ed +Ġnut rit +Ġland owners +Ġfl ushed +Ġbroad en +m ie +Ġprint ln +Ġn ig +ĠCorp us +J en +Ġprot o +ĠWik imedia +ĠPal o +C OR +Ġstory lines +Ġevangel icals +ĠDar rell +Ġrot or +ĠH W +sk illed +ery l +Ġbe gg +ĠBl umenthal +Ġwe aving +Ġdown wards +ĠJack et +ĠANG EL +Te chnology +Ġes oteric +alde hyde +Ġfur iously +Ġforeign er +We ak +CH O +ĠH ound +Exper ience +ĠPlay station +ĠM IA +ĠU ng +cl oth +ag all +Ġcal ming +iz ens +St ruct +ĠW itches +ĠCeleb ration +Ġ........ ...... +pt roller +ĠTC U +Ġb unny +ãĥ į +ut orial +Ġup scale +ĠSt a +ĠCol ossus +Ġchlor ide +ĠZ ac +ĠRe asons +ĠBrook ings +ĠWH ITE +][ / +ĠL ose +9 05 +Ġunders ide +ern els +Ġv ape +do zen +upp et +ĠST OP +mat ical +ĠStat ements +hed dar +P AC +Custom er +Ġmem os +ĠP J +end ars +ĠLim its +l augh +Ġstabil ized +ĠALE C +Y A +Up grade +al am +Ġtechn o +Ġan ew +fore seen +Ġcolleg iate +ĠPy ro +ĠD ism +Ġfront line +Ġammon ia +I U +Qu ite +John ny +ass in +G OP +ĠSt yles +ĠSovere ign +acter ial +5 49 +ĠR IP +ĠL ists +Ġ3 64 +ĠRece p +s ocket +ĠByr d +ĠCand le +An cient +Ġappell ant +en forcement +ace a +ans ki +Ġold s +88 6 +Ġsl urs +Ġem pires +Ġbuck le +Ġalien ation +ĠAber deen +Ġunic orn +Ġoverr iding +ĠL X +pp a +Ġdesp ised +ĠB ugs +ĠB ST +S outhern +5 33 +Ġhall mark +ĠPost er +Ġstem med +Ġprincip als +ĠT ECH +ĠSand wich +It aly +Ġche esy +ĠSet TextColor +ĠProt ective +ĠC ohn +J O +apt op +Re ason +Lead er +ĠUnder stand +ĠFr idays +ĠContin uous +Ġcl ipping +ĠR ye +Ġber th +tim er +ann is +re act +Ġbuff alo +ĠPar as +Ġ6 55 +Ġpres ided +ĠSun rise +Ġve ts +Ġcl oves +ĠMcC ull +Stre ngth +G AN +Ġill iter +ĠPric ing +l é +Ġresist or +Ġbr un +ĠSuff olk +Ñ ĭ +ĠL iver +Re leased +Ġwhat s +8 60 +ĠMe asures +Ġden ouncing +ĠRy zen +Ġsou ven +Ġcareg ivers +ch ini +ĠScar lett +Ġt rough +Cong ratulations +Ġtax is +ĠTrad ition +j it +Ġtable top +Ġhither to +Ġdis information +off ensive +h ra +ĠDISTR ICT +Ġcompl icate +chen ko +ĠRecon struction +Ġpalp able +Ġa usp +Ġ4 28 +Ġshowc ases +ĠPublic ation +know ledge +inn on +4 19 +Ġretri eval +and ers +Ġref ute +Ġinqu ired +g ur +Ġneg ativity +Ġcons erve +Ġafter life +Ġpres upp +ĠGill espie +Ġm t +ĠD N +T ap +Ġper pend +ĠS my +does n +Ġsp illing +Ġhyp ers +K ate +® , +ke pt +ĠP owered +Ġj a +ĠK lux +ard e +ab an +Ġ4 44 +Ġflatt ened +ĠImprove ments +urg a +ĠK und +Ġins cribed +Ġfac ult +Ġunpre pared +ĠCons umers +Ġsatisf ies +Ġpul monary +Ġinf iltration +Ġex ternally +Ġcongrat ulations +ag han +Ġair liner +Ġfl ung +Ġfly ers +G D +Ġsnipp ets +Ġrec ursive +Ġmaster ing +L ex +Ġovert ly +v g +Ġluck ily +Ġenc ro +ĠLanc et +ĠAbyss al +function al +Ġs ow +Ġsqu id +Ġnar ration +Ġn aughty +ĠHon our +ĠSpart ans +Ġsh atter +ĠTac oma +ĠCal ories +ĠR aces +Sub mit +Ġpurpose fully +w av +ĠY ok +F est +ĠG err +Met ro +Ġit iner +f amous +Ġ" { +in line +was her +Iss ue +ĠCL IENT +oz o +Vers ions +7 25 +ĠGl ock +Ġshield ed +ĠPC R +ENC Y +ĠWe ld +ĠSim pl +Ġredirect ed +ĠK ham +Ġ( > +Ġlab ou +Ġdi apers +ss l +Ġcell ar +organ isms +ore sc +ĠBer ks +did n +Sh ipping +C hest +Ġund one +Ġmillion aire +Ġc ords +ĠYoung er +appropri ately +Ġsequ els +u ve +ant icipated +Ġle wd +ĠSh irt +ĠDmit ry +V eter +Ġsl aying +ĠY ar +Ġcompl ication +I owa +ĠEric a +ĠBL M +g irlfriend +b odied +6 26 +19 63 +Ġintermedi ary +Ġcons olation +M ask +ĠSi em +ow an +Beg inning +Ġfix me +Ġculmin ated +Ġcon duc +ĠVolunte er +Ġpos itional +Ġgre ets +ĠDefin itions +Ġthink er +Ġingen uity +Ġfresh men +ĠMom ents +Ġ35 7 +ate urs +ĠFed Ex +s g +69 4 +Ġdwind ling +ĠBO X +sel age +Ġt mp +Ġst en +ĠS ut +Ġneighbourhood s +Ġclass mate +f ledged +Ġleft ists +Ġclim ates +ATH ER +ĠScy the +ul iffe +Ġs ag +Ġho pped +ĠF t +ĠE ck +ĠC K +ĠDo omsday +k ids +Ġgas ped +Ġmon iker +ĠL od +ĠC FL +t ions +r ums +fol ios +Ġm d +Ġunc anny +Ġtrans ports +ĠLab rador +Ġrail ways +Ġappl iance +ĠCTR L +æ Ģ +Pop ulation +ĠConfeder acy +Ġunb earable +Ġdors al +ĠIn form +op ted +ĠK ILL +Mar x +Ġhypoc ritical +q us +ĠN umerous +ĠGeorg ian +ĠAmbro se +ĠL och +Ġgu bernatorial +ĠX eon +ĠSupp orts +ens er +ee ly +ĠAven ger +19 65 +Ar my +Ġju xtap +Ġcho pping +ĠSpl ash +ĠS ustainable +ĠFin ch +Ġ18 61 +ict ive +at meal +ĠG ohan +Ġlights aber +ĠG PA +ug u +ĠRE PL +vari able +Ġher pes +Ġdesert s +ac iously +Ġsitu ational +week ly +ob l +Ġtext ile +ĠCorn wall +Ġcontrace ptives +ĠA ke +] - +ä¹ ĭ +: , +ĠW em +ĠB ihar +Ġ' . +Ġbe re +Ġanal ogue +ĠCook ies +Ġtake off +Whe el +Ġmaj estic +Ġcomm uting +0 23 +ĠCor pse +ass ment +min i +Ġgor illa +ĠAl as +ere e +Ġacquaint ances +ĠAd vantage +Ġspirit ually +Ġey ed +pm wiki +ĠE nder +Ġtrans lucent +Ġnight time +ĠIM AGES +5 45 +ĠK amp +ĠFre ak +Ġ ig +Port land +4 32 +ĠM ata +Ġmar ines +Ġh ors +ater asu +ĠAtt ribution +Ġ-------- - +Ġk ins +ĠBEL OW +++ + +Ġre eling +ol ed +Ġcl utter +ĠRel ative +Ġ4 27 +B US +Ġa vert +ĠChe ong +ĠA ble +ĠPry or +Develop er +Ġen cyclopedia +ĠUSA F +ĠG arry +Sp ain +Bl ocks +Ġexp osition +ĠGamer Gate +W OR +Ġstockp ile +Ġclot hed +ĠT one +ĠR ue +t umblr +Ġtreacher ous +Ġf rying +Ñ Į +ĠS ph +Ġrest raints +Ġemb odies +ĠG es +S afety +Ġnegoti ators +min ing +ĠAppalach ian +L OS +ĠJenn a +Ġpass ers +ç ĭ +sn ap +Ġshort en +creat or +Ġinn umerable +uther land +67 4 +ĠW OM +ĠAs cend +ĠArm ory +ĠTrans action +K ick +Ġsuit case +day Name +Ġwaste ful +mar riage +ĠMcC abe +ite ch +ĠO ss +Cl osure +ĠTreasure r +Ġindec ent +ĠD ull +Ġresid ences +19 59 +ĠS ettlement +Ham ilton +Ġself ies +ĠRank ing +ĠBark ley +ĠB ore +ĠW CS +ĠMar itime +ĠH uh +ĠForest ry +Ġcultiv ating +ĠBall ard +Ġg arrison +ĠSD L +9 30 +Ġnas cent +Ġirresist ible +Ġaw fully +\/ \/ +Ġequ ate +Ġanthrop ology +ĠSylv ia +Ġintest ine +Ġinnoc uous +cess ive +ag ra +ĠMet roid +G rant +8 55 +ģ ĸ +Ġ" _ +ãĥĥ ãĥī +Ġappra isal +ĠFred dy +04 6 +Ġ40 6 +Ġ18 30 +Ġd ocking +St atic +Ġp ont +ĠVolt age +ĠSt ead +ĠMort gage +ĠJon ah +Y L +CLASS IFIED +Ġas bestos +nik ov +Ġcoll agen +ĠOrb ital +P ocket +7 99 +Ġhy brids +inc hes +Ġinv oice +und y +Ġinequ alities +T rend +w ashed +B ALL +Ġluc id +ĠComment ary +Ġw itty +Br andon +Ġbru ising +Ġ6 20 +es cent +box ing +P OL +Ġ3 78 +R ect +Ġlic ences +ĠMcG ee +p ressed +D anny +Ġj ammed +ord inate +Ġle th +Ġdistingu ishes +ĠYam aha +IL S +ĠH ume +ĠC ategories +Rober ts +Ch art +Ġbeet le +ĠGra veyard +Ġ($ ) +o ÄŁ +Ġtw ilight +are lla +á ½ +Ġbooth s +ĠH HS +ĠFeld man +Ġexcav ation +Ġphilosoph ies +at ography +ĠGar age +te chnology +Ġunfor gettable +Ġver ifying +Ġsubord inates +E ls +Ġne b +G aming +EN A +ĠAchieve ment +it ters +ĠG abe +Ġd umps +for cer +Ġpo ignant +ĠM BA +ĠHe idi +ime i +Ġm ages +Ġliber ate +Ġcircum cised +ĠMer maid +ĠMat th +t ogether +ĠW ichita +Ġstore front +ĠAd in +V II +Four th +Ġexplore rs +W ER +Not able +Bro ok +m ens +F aith +-------- - +ĠJ ou +¬ ¼ +Ġpine apple +Ġam alg +el n +ark able +ĠãĤµ ãĥ¼ãĥĨãĤ£ +ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³ +Ġov arian +ĠE choes +Ġhairc ut +Ġp av +Ġch illed +anas ia +Ġsty led +Ġd ab +ni per +Ġminister ial +ĠD UP +T an +Ġsul ph +ĠD eter +ĠBo hem +od an +Ġeduc ator +â ĵĺ +sp ir +Ch icken +ĠE leanor +Ġqu i +Ġheav iest +Ġgrasp ed +U RA +Ġcro oked +Jess ica +pro blem +Ġpred etermined +Ġman iac +Ġbreath s +ĠLauder dale +Ġh obbies +y z +Cr ime +Ġcharism a +d L +Ġle aping +Ġk ittens +Ang elo +ĠJ ACK +ĠSu zanne +Ġhal ting +ENT ION +Ġswall owing +ĠEarthqu ake +Ġeight eenth +ĠN IC +ĠIN F +ĠCons cious +Ġparticular s +circ le +7 40 +Ġbene volent +Ġ7 47 +Ġ4 90 +Ġr undown +ĠVal erie +ĠB UR +Ġcivil isation +ĠS chn +W B +ot ide +intern ational +Ġj ohn +Ġ19 02 +Ġpe anuts +Ġflav ored +k us +Ġro ared +Ġcut off +é £ +Ġorn ament +Ġarchitect ures +Ġ3 69 +ol or +ĠWild e +ĠC RC +ĠAdjust ed +Ġprov oking +land ish +Ġrational ity +Ġjust ifies +Ġdisp el +Ġa meric +ĠPol es +Ø © +Ġen vis +ĠD oodle +ä½ ¿ +igs aw +auld ron +Techn ical +T een +up hem +ĠX iang +Ġdetract ors +ĠZ i +ĠJournal ists +Ġconduc ive +ĠVolunte ers +Ġs d +Know ing +Ġtrans missions +ĠPL AN +ĠL IB +Ġall uded +Ġob e +Ġd ope +ĠGold stein +Ġwavelength s +ĠDest ination +nd a +ug i +Ġattent ive +ĠLe an +ral tar +Ġman g +mb uds +ak ings +b ender +Ġacc ol +Ġcraw led +N OW +Min nesota +Ġflour ished +ĠZ up +ĠSuper visor +ĠOliv ier +Ex cellent +Ġwid en +D one +Ġw ig +Ġmiscon ceptions +Cor p +W an +Ġvener able +ĠNot ably +ĠKling on +an imate +Bo ost +ĠS AY +miss ing +ibli ography +mel on +Ġpay day +Ø ³ +bo le +Ġve iled +ĠAl phabet +It alian +Ġever lasting +ĠR IS +ĠC ree +rom pt +Ġh ating +Ġgrin ning +Ġge ographically +OS H +Ġwe eping +ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł +Ġimpe cc +Let ter +Ġblo ated +PL A +ĠFe in +Ġper sever +Th under +Ġa ur +ĠR L +Ġpit falls +âĸ º +Ġpredomin ant +Ġ5 25 +7 18 +AP E +7 14 +Ġfarm land +ĠQ iao +Ġv iolet +ĠBah amas +Ġinflic ting +ĠE fficiency +Ġhome brew +Ġundert ook +Ġcur ly +ĠHard ing +man ia +59 6 +Ġtem pered +Ġhar rowing +ĠP ledge +ĠFranken stein +è ª +M otion +Ġpredict ably +ĠExpl osion +oc using +er d +col o +FF ER +Ġback field +ĠV IDE +ue bl +N arr +ĠArg ument +Ġgen omic +Ġbout ique +Ġbatt ed +ĠB inary +Ġg amb +ĠRh ythm +67 3 +Ġa float +ĠOlymp ia +Y ING +Ġend if +is in +Ġwin ters +Ġsc attering +I v +D istance +Ġtr u +ĠCom fort +Ġne xus +Ġair flow +ĠByz antine +p ayers +con i +ĠB etsy +D eal +ĠN ug +ĠContin ent +red ibly +Ġoptim izing +al beit +Ġec static +ĠPro to +ç · +iv ot +âĸ Ħ +em p +rou nder +Ġcl out +ĠI ST +66 3 +ĠDoll ars +ĠD AC +Ġsubsc ribed +Ġrehears al +Ġam ps +ĠSh ang +es m +Ġspr inkle +Ġassail ant +ĠO o +ĠCoin base +T act +Ġret ina +Ġn uns +R ON +att o +Ġj ug +ĠSV G +Ġb ikini +ĠFI LE +ĠFound ers +ep ort +ĠK P +Ġrest ores +ĠTh ick +Ġash ore +Ġappro vals +R ender +M AG +G raham +ĠCort ana +ãĥ³ ãĤ¸ +ss h +or ians +ars ity +ĠInsp ired +u pper +Ġsign alling +Ġreb uke +Ġfl ares +Ġdownt ime +Stud ies +Ġstagn ation +ĠSequ ence +Ġgr unt +Ġass ures +ĠPL A +59 2 +Ġintra ven +d epend +Sus an +ĠManz iel +Man ia +Cont ract +Ġsl ams +Ġcult ured +Ġcred itor +L IST +ĠH UM +ĠChatt anooga +serv ed +Ġclo aked +ĠF TP +p owder +ĠSt ella +uct ive +Ġcheap ly +ĠMU CH +ĠGalile o +Ġsu ites +spe ech +Ġdeliber ations +ĠCh ips +« ĺ +Bal ance +ĠWyn ne +ĠAk ron +Ass et +Ġhon oured +Ġed ged +Like wise +anim ous +ĠW age +ĠEz ek +ad vertisement +ĠRT X +ĠM AD +Ġmigr ating +ĠS QU +Ġ4 75 +Ed ited +Ġshorth and +ĠBas ics +Ġcro tch +ĠEV EN +Ġv m +effic iency +Ġcal ves +ĠF rie +ĠBrill iant +Ġstri kers +Ġrepent ance +Ġarter ies +r l +B ed +h ap +Ġcrypt ography +ĠSab res +Ġ4 14 +vi ks +ih ara +aps es +T alking +Ġintertw ined +Ġdoc ks +Ġalle le +ĠArt ifact +ĠH IM +t orn +ç ķ +Ġop acity +ĠE ly +os uke +Ġn ipple +Ġhand written +ĠV K +ĠChamber lain +ĠLa os +ig raph +g row +Ġtr illions +Ġdescend ant +ĠSail or +as uring +Ġce ilings +ĠWare house +f lying +ĠGl ow +Ġn ont +Ġmiscar riage +Ġrig s +Ġmin istries +Ġelabor ated +Ġdel usional +ĠHum ane +Ġ3 79 +n ets +Ġblack out +add ers +Ġn p +ĠT ire +ro sc +Ġsub div +Ġlink age +Ġchron ological +ĠHER O +Ġres ettlement +ĠVin yl +Ġpast oral +ĠMob il +ĠBar bar +Co oldown +ĠF ritz +c riminal +re pe +Ġbell ig +ĠBre ed +Ġ4 18 +Ġsem blance +ij k +Ġcur tail +Ġclin ch +cont ained +ĠProm pt +ast on +Ġw i +Ġpursu its +5 15 +ĠGl oss +Ġfl ips +Ġcoup ons +Ġcl oning +ĠLike ly +Rem oved +ĠQu artz +r ices +ĠSpe ars +Ġp ious +Ġdep reciation +ĠD are +oun ces +am az +O nt +Ġp innacle +d ocker +0 26 +ĠW yr +ĠPro per +Ë Ī +n il +By tes +Ġseek er +t rial +Ġunf olds +ĠMar se +Ġextravag ant +ĠSurviv ors +RED ACTED +ĠSpeed way +ĠCra igslist +sub mit +ĠGener ations +Ġup holding +Ġblood stream +ĠMiss ions +ĠL awn +Ġlim bo +ene i +H uh +ĠWild cats +pre p +ĠMark us +ĠFor bidden +rit ic +IN O +Ġexhib iting +requ ent +ch uk +Ġhabit ual +ĠComp atibility +Dr ag +RIP T +uj ah +GR OUND +Ġdelinqu ent +Ġburn er +Ġcontempor aries +Ġgimm ick +load s +Ġno zzle +p odcast +ĠW ak +ĠStat en +ĠK uh +ãģ ĵ +inter rupted +Ġinv incible +ĠBurn ett +cig arette +ĠPeb ble +ĠTem porary +ĠMar ino +58 2 +Ġwast eland +ident ly +T x +Ġr ite +ĠPan asonic +ĠM iddles +ĠHort on +ae us +Ġc uring +Ġm ats +Ġadj ourn +Ġfears ome +pe z +bo ats +Ġpro pell +Ġconflic ted +ĠAng er +Ġinsurg ent +K arl +Ġco ales +Ġsouth western +Ġdis su +ĠO vert +******** **** +Ġbox ed +ĠBr une +aa a +Ġgard ening +ĠEng el +tr acks +Ġpur ified +Ġplace holder +ĠL ikes +Ġd an +G ab +Ġe ct +ĠF aw +ĠEl iot +Ġ' , +otrop ic +ĠRu in +hed on +Ġca ul +Ġa ft +ĠCad illac +gh a +ass ian +ud eb +ĠT ick +Ġadjust s +AR GET +5 37 +isc he +ant y +ĠFried rich +ĠBl izz +ĠA OL +Camp aign +Ġmamm al +ĠVe il +ĠK ev +ĠMaur it +ĠDam ien +N ation +E astern +Ġ{ : +Ġ= ================================ +Ġstereotyp ical +Ġatt ic +ĠCy borg +requ ire +Ġaward ing +ĠPap ua +bt n +b ent +B oo +Ġ( = +ĠX ander +ĠSomers et +Ġcatch y +Ġcert ify +STR UCT +Ġit al +Ġt ides +ĠBr ands +G ray +comp etitive +Ġcur ator +ĠD G +omin ium +ĠGM Os +ci ating +ĠCarm en +ow ard +Balt imore +Ġr gb +C u +Ġwip es +spe ll +IT NESS +Ġsummar izes +ĠRe vis +Ġwhistlebl owers +ĠBre ach +Ġcro chet +k os +ews ki +Ġrep et +Ġcrim son +ĠKar achi +read able +dim ension +ĠI gor +ild ed +ĠZ ed +ĠKe ane +ĠCos metic +DE P +Ġretreat ing +ĠU A +ens ical +Ġd usk +ĠDick ens +Ġaren as +ĠPass age +level s +Ġcur v +P ope +Ġch ores +ĠEl ise +ĠComp ass +b ub +Ġmamm alian +ĠSans krit +ĠAN C +ĠCr ack +Q ual +L aun +amp unk +Ġlearn ers +Ġglam orous +Ġfur the +erm ott +c and +Gener ic +Ġnarr ated +Ġdisorder ly +ĠTrans actions +ĠDet ention +ĠR oku +Ä į +Ġunder statement +ĠS aur +ĠRodrig o +ĠAS AP +S in +Ġre joice +Method s +Ġelectro de +Ġworsh ipped +Ġid i +ĠPhys icians +Ġpop up +Ġde ft +ĠRem oval +ĠBu enos +ver bs +Ġfun k +ush a +rict ion +ore a +ĠBang alore +ĠKen obi +zz i +Ġnorm ative +Ġgobl ins +Ġcaf es +ĠUN CLASSIFIED +ĠF ired +S IGN +Ġs clerosis +ĠV oter +ĠSon ny +ĠExt end +ĠEV s +Ar senal +Ġp si +Ġwid est +ĠT us +Ġlo oms +Ġjust ifying +ĠGr anger +è ¯ +Ref er +58 3 +Ġflour ishing +ab re +Ġr ave +ĠCont ra +Ġ18 98 +Add s +Ġf ul +ĠCo oke +some one += # +67 1 +Ġy ak +Ġar te +ĠMis cellaneous +ĠDet ection +ĠCl ancy +â ģ +ass ies +Ġval iant +ĠFemin ist +cor ruption +V el +P ear +Ġsucc inct +Ġquick est +k w +Ġsp itting +ĠL ibraries +åħ ī +ant z +D ad +ĠSpec ifications +rup ulous +and r +RES ULTS +Ġsnow ball +Ġpred is +ĠB axter +ĠNurs ing +ĠCh aff +s we +Ġout age +Ġnest ing +Ġnotor iety +tr igger +on ite +j on +Ġf ou +ook ed +ĠCelebr ity +re ality +Ġfat ig +Ġhug ging +Ġbother s +ĠPan zer +ĠCh andra +fig ured +Ġvol ts +ĠCloud s +Ġfee ble +ĠCur ve +ĠAs us +78 6 +abs or +ĠV ICE +ĠH ess +Ġmanufact ures +Ġgri zz +ĠPower ful +ac id +Ġsub sections +ĠKrug man +ĠAl ps +is u +Ġsequ est +ĠUlt ron +ĠT inker +ĠGo ose +Ġmism atch +Att orney +Ġmorph ology +ĠSix ers +ut tered +ĠE LECT +gr an +Rus sell +ĠG SL +Ġfort night +Ġ. ) +Ġapost le +pr one +el ist +Unt itled +ĠIm plementation +ist ors +Ġtank er +Ġpl ush +Ġattend ants +ĠT ik +ĠGreen wich +ĠY on +ĠSP L +cell s +unt led +S olution +ĠQu é +Ġvac ated +Ġupt ick +ĠMer idian +æ ĥ +ĠDr ill +9 25 +58 4 +Ġrenov ated +ĠKub rick +zy k +Ġl ousy +pp el +ohyd rate +ĠI zzy +lesi astical +CC C +ĠAj ax +Ġad apters +ĠPetra eus +Ġaffirm ation +ĠST OR +le ms +ad oes +ĠConstantin ople +Ġp onies +Ġl ighthouse +Ġadherent s +ĠBre es +omorph ic +Fight ing +Ġpl aster +ĠP VC +ĠOb st +Ġdear ly +ĠTo oth +icks on +Ġsh aming +P lex +A gg +Ġâ̦ " +Ġsub reddits +Ġpige on +ĠResident ial +ĠPass ing +Ġl um +ĠP ension +Ġpessim istic +Ġ4 32 +z inski +c ade +0 75 +Ġapolog ised +iy ah +Put ting +Ġgloom y +ĠLy me +=-=-=-=- =-=-=-=- +ĠT ome +ĠPsych iatric +ĠH IT +c ms +ap olog +Ġbreak er +Ġdeep en +Ġtheor ist +ĠHigh lands +Ġb aker +Ġst aples +Ġinterf ered +ĠAb ortion +jo ined +ch u +Ġform ulate +Ġvacc inations +Ġban ter +phe us +Ġoutfield er +ĠM eter +Ġ# #### +Ġ18 95 +Ġnarrow ing +ĠST ORY +f p +ĠC ST +ign ore +Ġproclaim ing +ĠR U +ĠB ALL +yn a +65 3 +Ġpos it +P RE +59 4 +ĠRegist rar +ĠPil grim +ic io +Ġpre tt +Ġlif eless +Ġ__ _ +Ne igh +ĠCh urches +orn o +Ġor cs +Ġkind red +ĠAud it +Ġmillenn ial +ĠPers ia +g ravity +ĠDis ability +ĠD ARK +W s +od on +Ġgrand daughter +ĠBro oke +ĠA DA +ER A +Ġpick ups +ĠWil kinson +ĠSh ards +ĠN K +Ġexp el +ĠKis lyak +Ġj argon +Ġpolar ized +ian e +Pub lisher +Ġreb utt +Ġapprehens ion +ĠK essler +Ġpr ism +F UL +19 64 +ĠL oll +ä ¿ +le thal +Å Ł +Ġg hetto +Ġb oulder +ĠSlow ly +ĠOsc ars +ĠInst ruction +ĠUl tr +ĠM oe +N ich +ĠP ATH +( * +ĠRE LEASE +un ing +rou se +en eg +Ġre imb +ĠDet ected +Do S +Ġster ling +Ġaggreg ation +ĠLone ly +ĠAtt end +hig her +Ġairst rike +ks on +SE LECT +Ġdef lation +ĠHer rera +C ole +rit ch +Ġadvis able +F ax +Ġwork around +Ġp id +mort em +ers en +Ġtyp o +Ġal um +78 2 +ĠJam al +script s +Ġcapt ives +ĠPres ence +ĠLie berman +angel o +Ġalcohol ism +ass i +Ġrec ite +Ġgap ing +Ġbask ets +ĠG ou +Brow ser +ne au +Ġcorrect ive +und a +sc oring +ĠX D +Ġfil ament +Ġdeep ening +ĠStain less +Int eger +Ġbu ggy +Ġten ancy +ĠMub arak +Ġt uple +ĠD roid +ĠS itting +Ġforfe it +ĠRasm ussen +ixt ies +es i +ĠKim mel +Ġmetic ulously +Ġap opt +ĠS eller +08 8 +ec ake +hem atically +T N +Ġmind less +Ġdig s +ĠAcc ord +ons ense +em ing +br ace +Ġe Book +ĠDist ribut +ĠInvest ments +w t +] ), +beh avior +56 3 +Ġbl inding +ĠPro testers +top ia +Ġreb orn +ĠKel vin +ĠDo ver +ĠD airy +ĠOut s +Ġ[ / +Ï Ģ +b p +ĠVan ity +ĠRec ap +ĠHOU SE +ĠF ACE +Ġ4 22 +69 2 +ĠAnt ioch +cook ed +Ġcoll ide +Ġa pr +Ġsle eper +ĠJar vis +Ġalternative ly +ĠLe aves +ĠM aw +Ġantiqu ity +ĠAdin ida +Ġab user +Poké mon +Ġass orted +ĠRev ision +ĠP iano +ĠG ideon +O cean +Ġsal on +Ġbust ling +ogn itive +ĠRah man +Ġwa iter +Ġpres ets +ĠO sh +ĠG HC +oper ator +Ġrept iles +Ġ4 13 +ĠG arr +ĠCh ak +Ġhas hes +Ġfail ings +Ġfolk lore +Ġab l +ĠC ena +ĠMac Arthur +ĠCOUR T +Ġperipher y +app ers +Ġreck oned +ĠInf lu +ĠC ET +Ġ3 72 +ĠDefin itive +ass ault +4 21 +Ġreservoir s +Ġd ives +ĠCo il +DA Q +Ġvivid ly +ĠR J +ĠBel lev +Ġec lectic +ĠShow down +ĠK M +ip ed +reet ings +ĠAs uka +L iberal +ĠÏ Ħ +Ġbystand ers +ĠGood win +uk ong +S it +ĠT rem +Ġcrim inally +ĠCirc us +ch rome +88 7 +Ġnan op +ĠOb i +ĠL OW +o gh +ĠAuth ors +ob yl +Ur ban +Ġt i +ĠWe ir +t rap +ag y +Ġparent heses +Ġout numbered +Ġcounter productive +ĠTob ias +ub is +P arser +ST AR +Ġsyn aptic +ĠG ears +Ġh iber +Ġdebunk ed +Ġex alted +aw atts +H OU +Ch urch +ĠPix ie +ĠU ri +ĠForm ation +ĠPred iction +C EO +Ġthro tt +ĠBrit ann +ĠMad agascar +ë ĭ +Ġbill boards +ĠRPG s +ĠBe es +complete ly +F IL +Ġdoes nt +ĠGreen berg +re ys +Ġsl ing +Ġempt ied +ĠPix ar +ĠDh arma +l uck +ingu ished +Ġend ot +Ġbab ys +05 9 +che st +r ats +Ġr idden +Ġbeet les +Ġillum inating +Ġfict itious +ĠProv incial +Ġ7 68 +Ġshe pherd +ĠR ender +Ġ18 96 +C rew +Ġmold ed +ĠXia omi +ĠSp iral +Ġdel im +Ġorgan ising +Ġho ops +ĠBe i +z hen +Ġfuck in +Ġdec ad +Ġun biased +am my +sw ing +Ġsmugg led +Ġk ios +ĠP ERSON +ĠInquis itor +Ġsnow y +Ġscrap ing +ĠBurg ess +P tr +ag ame +R W +Ġdro id +ĠL ys +ĠCass andra +Jac ob +Ġ35 4 +Ġpast ure +Ġfr anc +ĠScot ch +ĠEnd s +ĠI GF +def inition +Ġhyster ical +ĠBrown e +77 1 +Ġmobil ization +æ ķ +iqu eness +Th or +Ġspear headed +Ġembro iled +Ġconject ure +jud icial +Ch oice +Ġpaper back +P ir +Ġrec overs +ĠSur ge +ĠSh ogun +ĠPed iatrics +ãģ ł +Ġsweep s +ĠLabor atories +ĠP acks +al us +add in +Ġhead lights +g ra +Ev idence +COL OR +Ad min +Ĭ ± +Ġconco ct +s ufficient +Ġun marked +Ġrich ness +Ġdiss ertation +Ġseason ing +Ġg ib +ĠM ages +un ctions +ĠN id +che at +ĠTM Z +c itizens +ĠCatholic ism +n b +Ġdisemb ark +ĠPROG RAM +a ques +Ty ler +Or g +ĠSl ay +ĠN ero +ĠTown send +IN TON +te le +Ġmes mer +9 01 +Ġfire ball +ev idence +aff iliated +ĠFrench man +ĠAugust a +0 21 +Ġs led +Ġre used +ĠImmun ity +Ġwrest le +assemb led +Mar ia +Ġgun shots +ĠBarb ie +Ġcannabin oids +ĠTo ast +ĠK inder +IR D +Ġre juven +Ġg ore +Ġrupt ure +Ġbre aching +ĠCart oon +Ġ4 55 +ĠPale o +6 14 +Ġspe ars +ĠAm es +ab us +Mad ison +GR OUP +Ġab orted +y ah +Ġfel on +Ġcaus ation +Ġprep aid +Ġp itted +op lan +ĠShel ley +ĠRus so +ĠP agan +Ġwill fully +ĠCan aver +und rum +ĠSal ary +ĠAr paio +read er +ĠR ational +ĠOver se +ĠCa uses +Ġ* . +Ġw ob +Ke ith +ĠCons ent +man ac +77 3 +6 23 +Ġfate ful +et imes +Ġspir ited +ĠD ys +Ġhe gemony +Ġboy cot +ĠEn rique +em outh +Ġtim elines +ĠSah ara +ĠRel ax +ĠQuin cy +ĠLess ons +ĠE QU +SE A +N K +ĠCost co +Incre ase +Ġmotiv ating +ĠCh ong +am aru +ĠDiv ide +Ġped igree +ĠTasman ia +ĠPrel ude +L as +9 40 +57 4 +Ġch au +ĠSp iegel +un ic +-- > +ĠPhil ips +ĠKaf ka +Ġuphe aval +Ġsent imental +Ġsa x +ĠAk ira +ser ial +Mat rix +Ġelect ing +Ġcomment er +ĠNeb ula +ple ts +ĠNad u +ĠAd ren +Ġen shr +ĠR AND +fin ancial +ĠCly de +uther ford +Ġsign age +Ġde line +Ġphosph ate +rovers ial +f ascist +ĠV all +ĠBeth lehem +Ġfor s +Ġeng lish +S olid +N ature +Ġv a +ĠGu ests +Ġtant al +Ġauto immune +;;;;;;;; ;;;; +ĠTot ally +ĠO v +Ġdef ences +ĠCoc onut +Ġtranqu il +Ġpl oy +Ġflav ours +ĠFl ask +ãĤ¨ ãĥ« +ĠWest on +ĠVol vo +8 70 +Ġmicro phones +ver bal +R PG +Ġi ii +; } +0 28 +Ġhead lined +Ġprim ed +Ġho ard +ĠSh ad +ĠEN TER +Ġtri angular +Ġcap it +l ik +ĠAn cients +Ġl ash +Ġconv ol +Ġcolon el +en emy +G ra +Ġpub s +ut ters +Ġassign s +ĠPen et +ĠMon strous +ĠBow en +il ver +H aunted +ĠD ing +start ed +pl in +Ġcontamin ants +ĠDO E +ff en +ĠTechn ician +R y +Ġrob bers +Ġhot line +ĠGuard iola +ĠKau fman +row er +ĠDres den +ĠAl pine +E lf +Ġf mt +ĠS ard +urs es +g pu +Un ix +Ġunequiv ocally +ĠCitizens hip +qu ad +m ire +ĠS weeney +B attery +6 15 +Ġpanc akes +Ġo ats +M aps +ĠCont rast +mbuds man +ĠE PS +Ġsub committee +Ġsour cing +Ġs izing +ĠBuff er +ĠMand atory +Ġmoder ates +ĠPattern s +ĠCh ocobo +ĠZ an +ĠSTAT ES +ĠJud ging +ĠIn her +* : +Ġb il +ĠY en +Ġexh ilar +oll ower +z ers +Ġsn ug +max imum +Ġdesp icable +ĠP ACK +ĠAn nex +Ġsarcast ic +Ġlate x +Ġt amp +ĠS ao +b ah +ĠRe verend +ĠChin atown +ĠA UT +d ocumented +ĠGA BA +ĠCan aan +ĠÙ ħ +Ġgovern s +pre v +E sc +ĠEst imates +OS P +Ġendeav our +ĠCl osing +omet ime +every one +Ġwor sen +Ġsc anners +Ġdev iations +ĠRobot ics +ĠCom pton +Ġsorce rer +Ġend ogenous +Ġem ulation +ĠPier cing +ĠA ph +ĠS ocket +Ġb ould +ĠO U +ĠBorder lands +Ġ18 63 +G ordon +ĠW TO +Ġrestrict s +Ġmosa ic +Ġmel odies +ç Ħ +T ar +Ġdis son +ĠProv ides +Ġ ...... +b ek +F IX +Ġbro om +ans hip +Do ctors +Ġner ds +ĠReg ions +na issance +Ġmet e +Ġcre pt +pl ings +Ġgirlfriend s +kn it +ig ent +ow e +Ġus hered +ĠB az +M obil +4 34 +ĠPres ents +orig in +Ġins omnia +ĠA ux +4 39 +ĠCh ili +irs ch +G AME +Ġgest ation +alg ia +rom ising +$ , +c row +ĠIn spection +at omic +Rel ations +J OHN +rom an +ĠClock work +ĠBak r +m one +M ET +Ġthirst y +Ġb c +Ġfacult ies +R um +Ġnu ance +ĠD arius +ple ting +fter s +etch up +Reg istration +ĠK E +R ah +Ġpref erential +ĠL ash +ĠH H +Val id +ĠN AV +Ġstar ve +ĠG ong +z ynski +ĠAct ress +Ġw ik +Ġun accompanied +lv l +Br ide +AD S +ĠCommand o +ĠVaugh n +Wal let +Ġho pping +ĠV ie +Ġcave ats +Ġal as +if led +ab use +66 1 +Ġib n +Ġg ul +Ġrob bing +t il +IL A +Ġmit igating +Ġapt ly +Ġty rant +Ġmid day +ĠGil more +ĠDe cker +Ġ§ § +part ial +Ex actly +Ġphen otype +Ġ[+ ] +ĠP lex +ĠI ps +vers ions +Ġe book +Ġch ic +g ross +":" "},{" +ĠSur prisingly +M organ +Ġresid ues +ĠConf ederation +in feld +Ġl yr +mod erate +Ġperpend icular +V K +Ġsynchron ized +Ġrefres hed +Ġad ore +ĠTor ment +ol ina +Ġ26 00 +Item Tracker +Ġp ies +ĠF AT +ĠR HP +0 48 +ĠRES P +ĠB J +all ows +P and +Ġunw elcome +ĠV oc +ĠBast ard +ĠO W +ĠL AR +ĠHeal er +Environment al +ĠKen yan +ĠTr ance +ĠP ats +Ġali ases +ĠGar field +Ġcampaign er +Ġadvance ments +ĠOkin awa +ĠC oh +ows ky +Ġstar ved +Ġsize able +Ġ: -) +Ġm RNA +Ġsusp ensions +ist ar +Scot land +Pr in +-------------------------------- ---------------- +Ġ50 2 +Ġteasp oons +Ġ10 50 +Ġcoerc ive +ĠMason ic +edd ed +ĠPass enger +Ġl att +Ġbr aces +ĠSt eal +ĠNY T +ĠK ats +ĠCel est +ae z +T u +ĠCoul ter +ðŁ ĺ +Fl ickr +ĠWil mington +ith s +++ ; +Ġv ending +Ġneg ro +ĠPh i +ĠYellow stone +Call back +Ġsh ampoo +ĠSh ades +w at +Ġsuper human +Ġridic uled +Ġhol iest +om bo +Ġintern s +Ġh one +ĠPar agu +UR I +Ġd angling +ãĤ » +so v +ict ional +av ailability +Ġrev ocation +Ġd ow +in ic +ĠTHE IR +Ġis o +Ġout ings +ĠLeth al +Ġ) )) +Ġinacc ur +Ġout landish +Ġan us +let ico +id on +l ol +Ġun regulated +Ġsuccumb ed +Ġc uff +ĠWast eland +let al +Ġsub str +Ġcoff ers +Ġautom akers +ov i +ĠX ue +ĠDayton a +Ġjar ring +Ġf umes +Ġdisband ed +z ik +itt on +Ġstriking ly +Ġsp ores +Ad apter +.) : +ĠLynd on +ival ry +Ġor ally +Ġtumult uous +Ġdisple asure +Ġcon es +or rect +Ġappe ase +Ġder by +ĠTrip oli +ĠAl ess +Ġp oked +ĠGu ilty +v P +En ough +Ġorig inals +6 99 +Ġrabb i +Ġproverb ial +Ġpostp one +el ope +ĠMist y +Ġstaff ed +ĠUn employment +redit ary +Ġdilig ent +re comm +me asures +as in +8 25 +Ġpond s +Ġmm ol +ĠS AR +ĠC ARE +Ġ3 71 +Ġclen ched +ĠCors air +Ġcaric ature +z n +att ach +ĠSch ro +spe ak +p ainted +ĠS uc +ĠE NT +Ġcell ul +ĠP aid +di agn +WH ERE +Ġtext ed +B arn +Ġret racted +ĠRe ferred +S av +Ġup keep +Ġwork places +ĠTok ens +Ġampl ify +cl inical +Ġmult ic +mber g +Ġconvol uted +Reg ion +5 65 +ĠTop ic +Ġsn ail +Ġsal ine +Ġins urrection +ĠPet r +f orts +B AT +ĠNav ajo +Ġrud imentary +ĠLak sh +OND ON +Me asure +Ġtransform er +ĠGodd ard +Ġcoinc ides +ir in +R ex +ĠB ok +qu it +Ġshotgun s +Ġprolet arian +Ġsc orp +ĠAd a +5 14 +Ġsl ander +record ed +Ġemb ell +ris ome +Ġapolog izing +ĠMul cair +ĠGib raltar +Cl a +Ġall ot +ĠAtt ention +Ġ4 33 +le ave +Ġwh ine +ĠIss a +ĠFa ust +ĠBar ron +hen y +Ġvictim ized +J ews +Ġnurt uring +ett el +W inged +ĠSub tle +Ġflavor ful +ĠRep s +eng ed +call back +Ġdirection al +Ġcl asp +ĠDirect ions +plan et +icult ure +Hel per +ic ion +ac ia +Ġç ¥ŀ +Ġsur ges +Ġcan oe +ĠPrem iership +be en +Ġdef ied +ĠTro oper +Ġtrip od +Ġgas p +ĠE uph +ĠAd s +vern ight +high ly +R ole +Ġent angled +ĠZe it +6 18 +ĠRust y +Ġhaven s +ĠVaugh an +HA EL +ĠSER VICE +/ , +Ġstr icken +Ġdel usions +Ġb is +ĠH af +Ġgrat ification +Ġent icing +UN CH +Ad ams +ĠOL ED +ĠBeet le +Ġ18 99 +ĠSO FTWARE +ateg or +V L +ĠTot em +ĠG ators +AT URES +Ġimped ance +Reg istered +ĠC ary +ĠAer ial +on ne +en ium +Ġd red +ĠBe g +Ġconcurrent ly +Ġsuper power +ĠX an +j ew +imes ter +ĠDick inson +âĶ ģ +F la +Ġp ree +ĠRoll ins +© ¶æ +Ġden omination +ĠL ana +5 16 +Ġinc iting +sc ribed +j uries +ĠWond ers +app roximately +Ġsusp ending +Ġmountain ous +ĠL augh +oid al +N s +Det ect +) = +ĠL uthor +ĠSchwarz enegger +ĠMull er +ĠDev i +ec ycle +J ar +6 13 +ĠL ongh +B ah +ĠSP ORTS +n w +Ġref inement +Ġwater ways +Ġd iner +Bl ade +68 3 +F ac +Ġinitial s +Ġro g +Ġparan ormal +B UT +Ġ[ ( +ĠSw anson +ĠM esh +âĸ ¬ +Impro ve +ĠRad iation +ĠEst her +ĠE sk +ĠA ly +ik y +Ġir rad +ĠBuck ingham +Ġref ill +Ġ. _ +Re pe +CON CLUS +Ġdifferent iated +Ġchi rop +ĠAt kins +Pat tern +Ġexc ise +Ġcab al +N SA +ĠST A +ĠS IL +ĠPar aly +Ġr ye +ĠHow ell +ĠCount down +ness es +alys ed +Ġres ize +ãĤ ½ +Ġbudget ary +ĠStr as +w ang +Ġap iece +Ġprecinct s +Ġpe ach +Ġsky line +Ġ35 3 +pop ular +App earances +ĠMechan ics +ĠDev Online +S ullivan +Z en +Ġp u +op olis +5 44 +Ġde form +Ġcounter act +ĠL ange +Ġ4 17 +Con sole +77 4 +Ġnodd ing +Ġpopul ism +Ġhe p +Ġcoun selling +compl iance +U FF +Ġunden iably +Ġrail ing +ĠHor owitz +ĠSim one +ĠBung ie +Ġa k +ĠTal ks +x ff +fl ake +Cr ash +Ġsweat y +Ġban quet +ĠOFF IC +Ġinvent ive +Ġastron omer +ĠStam ford +ĠSc are +ĠGRE EN +olic ited +Ġr usher +Ġcent rist +ight ing +Ġsub class +Ġdis av +Ġdef und +ĠN anto +oci ate +m ast +Ġpac if +Ġm end +e ers +imm igration +ESS ION +Ġnumber ing +Ġlaugh able +ĠEnd ed +v iation +em ark +P itt +Ġmetic ulous +ĠL F +Ġcongrat ulated +ĠBir ch +Ġsway ed +Ġsemif inals +Ġhum ankind +m atter +ĠEqu ip +opa usal +S aid +ĠLay out +Ġvo icing +Ġth ug +Ġporn ographic +I PS +Ġmo aning +Ġgriev ance +Ġconf essions +esc al +TEXT URE +Aut hent +os aurus +P urchase +Ġreleg ation +al ter +ĠÂł Âł +Ġr iddled +Ġo gre +ĠLow ell +Occ up +E at +ĠHy der +ĠAdvis er +Com merce +H unt +ĠOr th +ĠComp etitive +ĠCL A +CD C +Ġsal ads +F le +Ġindustrial ized +` , +ĠO WN +Ġbec k +ĠPart icularly +oub t +Ġm M +ĠHuss ain +ĠChen nai +Ġ9 20 +Ġappoint ing +ĠCull en +,,,, ,,,, +Ġp ores +ver ified +Ġbi ochemical +em ate +Ġcoward ly +ĠHels inki +ĠEthiop ian +S OURCE +ER C +est ro +Ġbi otech +ĠS our +Ġbrew er +Bloom berg +Ġintens ify +Gl ass +an co +ĠF DR +gre SQL +ĠF ires +©¶æ ¥µ +ec o +100 1 +ĠHom eless +Ġinstant aneous +ĠH aste +ig el +D iamond +Ġp aving +Ġland fill +Ġd ads +h oun +: ] +Ġinc endiary +ĠLiving ston +ĠHil bert +ĠChe cks +st yles +in ators +ĠCl ive +ph rine +Ġchimpan zees +Ġp all +ĠJ M +ĠAad haar +ð Ŀ +Ġachie vable +dis abled +P ET +OOOO OOOO +M ot +Ġint angible +Ġbal let +ĠWe bs +ĠEst imated +Effect s +Ġb ailed +Josh ua +Ġturb ulence +Ġoccup ant +ĠDay light +Ġ36 1 +me et +Ġstat ically +Ġon look +Ġk i +il legal +Ġvel vet +Ġdehyd ration +Ġacqu ies +ĠRe z +ak ura +ĠU pton +at ro +Ġincomp rehensible +Ġback door +ĠRh ino +7 27 +Ġmath s +) + +Ġhe resy +Ġd f +ĠRoc he +ĠL ydia +Ġpanc reat +re ply +arre ll +Ġsolicit ation +Ġcirc adian +BI P +Ġfor ay +Ġcrypt ic +iz u +ime o +ĠTom ato +ĠH oms +ex amination +Ġqu arry +ĠVal iant +ĠJer icho +ĠIN CLUD +Ġ18 40 +5 19 +Ġres ists +Ġsnap shots +ĠSp ur +ĠAnt iqu +Log in +Ġbest selling +Ġant ic +ĠS utherland +ãĤ¢ ãĥ« +Ġ~ / +ĠP arm +è ĥ +P ages +int ensity +Ġimm obil +Ġ18 65 +zz o +Ġn ifty +Ġf entanyl +ĠPres ervation +op hen +Ġd arts +ĠD inosaur +po inters +ĠR ite +s uggest +aware ness +ĠSher idan +Ġst ances +Ġsor cery +Ġper jury +ĠNik ola +ie ver +Ġf iance +ĠJordan ian +ĠBall oon +Ġn ab +Ġk b +Ġhuman ities +ĠTan aka +hill ary +Ġconsult ancy +ĠZ ub +Ġrem ission +Ġconf id +CH Q +ĠF ug +Ġimpro vis +Y ep +/ _ +Ġunwilling ness +Ġport folios +05 5 +ĠInstruct or +aim an +Ġclaim ants +M bps +ĠBy e +re ceived +T weet +Ġind emn +ri z +am ara +N at +Ġeval uates +ĠL ur +ep ad +FO X +ĠTh ro +Ġrust y +Ġbed rock +ĠOp rah +J B +Ġmanip ulative +Ġwill ful +Ġrel apse +Ġext ant +The me +S ensor +ĠSt ability +go vern +Ġpo ppy +Ġkn ack +Ġins ulated +ĠT ile +ĠExt rem +Ġunt old +Ġconver ge +Ġref uel +ig roup +Ġdistort ions +Ġrav aged +Ġmechan ically +ĠRe illy +ĠN ose +ĠIncarn ation +ĠBeck y +abb ling +Ġt aco +Ġr ake +Ġmelanch oly +Ġillust rious +ĠDart mouth +Gu ide +ĠR azer +ĠBen z +Ult imate +ĠSur prise +Ġpage ant +off er +Who ever +Ġw iser +Ġchem ist +ĠHE LL +ĠBul k +Ġpl utonium +ĠCO VER +Ö ¼ +f ailed +Ġtire lessly +Ġinf ertility +ĠTr ident +ĠShow time +ĠC iv +V ice +requ ires +itt ance +Ġun controlled +interest ing +56 1 +Ġinnov ate +ateg ic +L ie +ĠS elling +U l +Ġsav ior +ĠT osh +Ġsw ast +P ASS +Ġr ink +Ġcard io +ĠI ro +ud i +Ġv antage +Ġv ans +ĠNi ño ++ = +Ġpropag ate +< ? +Ġmethod ological +204 39 +Ġtrig lycer +Ġing rained +ĠAn notations +arr anted +6 17 +ĠS odium +ĠA AC +techn ical +mult ipl +Ġ3 73 +å ĭ +Ġdec isively +Ġboost ers +Ġdessert s +ĠGren ade +Ġtest ifying +ĠSc ully +ID s +Ġlock down +ĠSc her +ĠR é +ĠWhit man +ĠRams ay +rem ote +Ġh ikers +ĠHy undai +Ġcons cientious +Ġcler ics +ĠSiber ian +ut i +is bury +Ġrel ayed +Ġqu artz +ĠC BI +seek ers +ull a +Ġweld ing +ĠSh al +ble acher +T ai +ĠSam son +Ġt umble +ĠInvest or +Ġsub contract +ĠShin ra +ow icz +j andro +d ad +Ġtermin ating +ĠNe ural +ä» £ +Ġleak age +ĠMid lands +ĠCaucas us +í ķ +c it +ll an +iv ably +ĠAlb ion +Ġ4 57 +Ġregist rations +Ġcomr ade +Ġclip board +0 47 +Ġdiscour aging +ĠO ops +Ad apt +Ġem path +n v +ĠPR OT +ĠDon n +ĠP ax +ĠB ayer +t is +Squ are +Ġfoot prints +part icip +ĠChile an +B rend +ind ucing +M agn +Ġclub house +ĠMagn um +Ġenc amp +ĠEth nic +uch a +ere y +Ġw atered +ĠCal ais +Ġcomplex ion +Ġsect s +Ġren ters +Ġbr as +oÄŁ an +Time out +Man agement +Ġinf ographic +P okemon +Cl ar +Ġloc ality +Ġfl ora +as el +P ont +Ġpop ulate +ĠO ng +Ġsubs istence +Ġa uctions +ĠMcA uliffe +ĠL OOK +br inger +Ġtit an +Ġmanif old +ĠâĹ ı +Ġcalibr ated +Ġcal iphate +ĠSH E +ĠCommission ers +ce ivable +j c +W inner +5 24 +Ġcond one +Other wise +Ġp iling +Ġem body +ĠCrime an +ut ics +ĠEx hibition +Ġ4 26 +e ering +Ġv ying +ĠH UGE +* =- +Ġprin cipled +à ¦ +Ġquir ks +ĠEdit ors +put ing +G ES +ĠF TA +ठ¾ +add on +ĠH AM +ĠFrie za +W oman +. $ +Ġc rib +ĠHer od +Ġtim ers +ĠSp aces +ĠMac intosh +at aka +Ġgl ide +Ġsmell ing +ĠB AL +Ġun su +Ġcond os +Ġbicy cl +ĠRev ival +55 3 +Ġjugg ling +H ug +ĠKardash ian +ĠBalk ans +mult iple +Ġnutrit ious +oc ry +19 00 +Ġinteg rates +Ġad joining +ĠF older +roll ment +ven ient +Ġu ber +y i +Ġwh iff +ĠJu ven +ĠB orough +net te +Ġb ilingual +ĠSp arks +ph thal +man ufact +Ġt outing +ĠPH I +Ke efe +Rew ard +Ġinf all +ĠTem per +typ ically +ĠNik ol +Ġregular s +Ġpseud onym +Ġexhib itions +Ġbl aster +Ġ40 9 +w arming +Ġrever ber +Ġrecip rocal +Ġ6 70 +ip ient +b ett +ĠBe gins +Ġit ching +ĠPh ar +Ass uming +Ġem itting +ĠML G +Ġbirth place +Ġt aunt +ĠL uffy +ĠAm it +Ġcir cled +ĠN ost +enn ett +Ġde forestation +ĠHist orically +ĠEvery day +Ġovert ake +79 2 +Ġn un +ĠLuc ia +Ġaccompan ies +ĠSe eking +ĠTr ash +an ism +R ogue +Ġnorth western +ĠSupplement al +ĠNY U +ĠF RI +ĠSat isf +x es +5 17 +Ġreass ured +Ġspor adic +Ġ7 01 +Ġmed ial +Ġcannabin oid +Ġbarbar ic +Ġep is +ĠExplos ive +ĠD ough +Ġuns olved +Support ed +Ġacknowled gment +sp awn +Ġkit chens +Ġ- = +talk ing +ic ist +ĠPeg asus +ĠPS U +Ġphot on +ĠAuthent ication +R G +@# & +76 2 +ĠCl air +Ġdi aper +Ġbr ist +ĠProsecut ors +ĠJ em +6 28 +ĠEvery where +ĠJean ne +equ ality +ãĥ© ãĥ³ +object s +ĠPel icans +Ġ39 2 +Ġbl u +b ys +ĠA go +Ġinstruction al +Ġdiscrim inating +ĠTR AN +ĠCorn el +ag os +Ġty re +Ġas piration +ĠBrid gewater +": - +! ". +ĠEn s +ĠCoc o +P ie +Ġdet ach +ĠC ouch +Ġphys ique +ĠOccup ations +osc opic +en ough +B uzz +App earance +Y P +Ġrac er +Ġcompl icity +r pm +T oy +Ġinterrupt s +ĠCat alyst +Ġut ilitarian +imp act +Ġsp aghetti +Ġp orous +Ġeste emed +Ġinc iner +ĠI OC +7 48 +Ġesp resso +ĠSm ile +abil ia +6 35 +Ġmathematic ian +Ġ4 24 +ĠK L +ĠH IP +Ġover heard +ĠT ud +ĠT ec +Ġqu izz +Ġfl attering +Ġcon n +âĢ İ +Ġatt aches +ĠR OS +ĠAC S +Ġt cp +ĠSh ame +sk ip +res pected +ĠTrin idad +gr ain +Ġfooth old +ĠUnch arted +ĠJul io +z l +av ored +ĠAn xiety +er rors +ĠCent auri +its ch +D addy +Ġclutch ing +ĠIm plement +ĠGut ierrez +Ġ7 60 +Ġtele portation +end ra +Ġrevers ible +st ros +Ad venture +08 3 +Ġliber ating +Ġas phalt +ĠSp end +AR DS +im sy +PR ES +ĠEmer ging +Ġwild fires +Ġtechn ologically +Ġem its +ĠART ICLE +Ġirregular ities +Ġcher ish +çī Ī +Ġst ink +ĠR ost +Econom ic +Ġcough ing +ĠMcC ann +pro perties +ilant ro +Ġreneg oti +Trans lation +Ġin quest +ĠGra pe +oot ers +gu i +ĠSwords man +ace ae +h itting +Ġr c +Ġexert ed +ĠS AP +it ent +Ġperil ous +Ġobsc urity +Ġassass inate +Ġab original +Ġresc uing +ĠSh attered +lock ing +all ion +Ch anging +ĠHar rington +ĠB ord +ĠAfgh ans +Jam ie +aret z +ĠAugust us +Ġ38 6 +8 30 +Ġj og +ok ingly +Tr igger +ĠH OR +Stat istics +Ġviewers hip +Ġadd itives +h ur +Ġmaxim izing +ĠR ove +ĠLou ie +ĠBuck et +ĠCHR IST +ou sel +Ġstre aks +ir ted +Ġt ert +Ġcolonial ism +Ġbur ying +y k +Cond ition +ĠDPR K +By Id +75 1 +âĹ ¼ +Ġwor risome +Ġvoc ational +sl ice +Ġsa ils +ĠCorrection al +95 4 +Ġt ul +K id +l uster +Ġfam ilial +ĠSp it +ĠEp iscopal +Specific ally +ĠVol cano +run s +q s +Ġve tted +Ġcram med +t rop +here r +Thank fully +Ġper cussion +Ġor anges +Ġround up +Ġ4 99 +x ious +Char acters +ĠZion ism +ĠR ao +ÃĽ ÃĽ +W F +Ġunintention al +ONE Y +Gr ab +Com mercial +Ġglut amate +ĠMcK enna +ru ciating +ning ton +ih u +Ch an +ĠSw ap +Ġleaf lets +Ġfunction ally +er ous +F arm +Ġcal oric +ĠLiter ally +con cert +Ġshe nan +Ġrep aid +ey es +Ġbas hing +ĠG orge +Ġcollabor ations +Ġun account +itch ie +Ġteam work +pp elin +Ġpip ing +Ġmin ced +Ġd iam +ri eg +Ġmasc ara +Ġsuck er +ĠMo ons +App s +ĠPe ck +Ġper v +ĠFl oat +o ley +ĠN ish +im ize +Ġarom atic +u in +end ish +! / +ĠB icycle +ĠAS IC +ile ged +ĠQuad ro +ios yn +Ġlock out +ĠW ink +SP EC +Attempt s +Ġseed ed +red o +ias is +Ġsn ag +ãĥķ ãĤ© +ãĤ ¶ +Ġground ing +Ġrelie ver +Ġfrivol ous +ĠG ifts +ĠF aces +Es pecially +Ġmicrobi ome +im ag +ĠSch l +ĠP les +ĠBle ach +ĠIr win +ĠE aton +ĠDisc iple +Ġmultipl ication +Ġcoer ced +Ġ4 19 +st h +E vil +B omb +Ġex orc +Ġstag gered +L ESS +Ġinert ia +ĠED IT +Ġgo b +Tr aditional +Ġclass y +Lear y +ĠP AGE +yr s +Ġtrans porter +Ġmat ured +Ġhij ab +Ġbi ome +Where as +Ġex termination +ĠT ues +ĠT akeru +ĠAud rey +er ial +ĠAd en +aff les +Ġnarciss istic +ĠB aird +UT F +I re +ĠCon nie +Ch amp +Ġwhis pering +ĠH att +D K +Ġdis infect +Ġdeduct ed +Ġpart ake +Ġdown grade +ĠEs ports +ĠContin uing +Ġdemocr atically +icro bial +itt a +Ġlim estone +Ġexempt ed +ĠFren zy +H erm +7 28 +Ġfled gling +Met a +765 61 +69 3 +% : +w ake +5 26 +ĠDis cipline +Ġvirgin ity +ĠLeg ions +ĠFrank ie +int ent +Ġrest rooms +ĠRou ter +da q +Ġobjection able +âĨ ij +w ark +ĠRah ul +g ain +activ ation +abs olute +ĠAccess ed +Ġ24 00 +ogg les +Ġsecond ly +ĠDEF ENSE +Ġpost age +wra pper +sh arp +7 29 +Ġcommun icates +Ġadd on +ĠMil itia +H ong +Ġsl umped +ĠJP EG +ĠI car +ad ish +68 1 +Ġmaj esty +ĠWolf gang +ĠEl astic +u per +Ġv iz +Ġunconscious ly +ĠST D +ĠS ass +Ġflower ing +ĠHel ic +ĠDra per +ĠAm ateur +Ġman ure +Ġdis ingen +ĠLe i +br ing +9 49 +Ġinhib ited +Ġhead quartered +Ġen igmatic +�� � +Ġred ress +R H +Ġratt led +Ġd iction +l io +ĠT BA +ĠSN AP +C alling +Ġfasc ists +ĠD ove +iew icz +0 36 +Ġco asts +ĠR ect +Ġ) ] +L ot +6 29 +ĠS EM +ĠPeters en +ĠExpl ain +ĠBo ards +ĠBe zos +ĠJ ournals +Ġ20 24 +p arser +Ġmist rust +Ġgr ate +ĠL ocked +bo a +S aint +g aming +Ġvow el +in ately +bl ow +All ah +Ġun matched +Ġb ordering +ĠExp end +n r +Or acle +rou ch +Ġcont iguous +ac us +Ġdist raught +58 1 +Ġanat omical +O X +ap ixel +8 33 +ĠPL US +Ġres usc +Ġab iding +57 3 +Ġvac ancies +Em ily +Ġhyp othal +ĠWer ner +ĠWe e +ĠDJ s +5 13 +Ġwitch craft +Ġac upuncture +ent ary +benef it +Product s +ĠP SP +ĠMP G +ĠJ inn +ĠJ arrett +Ġ4 45 +ĠIm aging +ĠP yth +Fin ish +Ġte x +Ġjuven iles +Ġhero ism +Ġdoubt less +ĠA ki +ĠT end +ĠPatri arch +Ġbit ters +ĠTele communications +it atively +ag na +Ġr g +ĠS OLD +Ġcomp ulsion +ĠN asa +ĠKath ryn +Ġmillion aires +Ġintrins ically +Ġbolst ered +time out +fl o +Ġtut or +p our +Stat ement +Ġ{ * +ĠRud olph +ĠKimber ly +rog ens +adi q +] + +Ġindign ation +Ġfract uring +ĠRe leases +ĠGr ain +pro tein +L ago +Ġvac ations +Ġboot ed +ĠTH REE +ĠH G +oresc ence +Ġt f +Ġso ar +iosyn cr +Ġgl ances +ĠSp oon +ĠJ ury +ĠCow boy +Ġcreat ively +Hig her +Ġsolic itor +Ġhaw k +ac io +89 6 +Ġsuperf lu +Ġbombs hell +ct ure +Ġbroker age +Ġraid ing +Ġf rench +Ġang led +Trans action +ĠGen ocide +u pe +ĠHait ian +57 2 +! : +Ġunwitting ly +iter ator +sc roll +Ġtall ied +Ġbi omedical +ĠC ARD +Ġe uphem +Ġbrain storm +a quin +K o +Mic helle +ĠR unes +ĠBall istic +ud ers +Ġmod esty +ĠiP ads +ĠEzek iel +Y E +Ġstars hip +Ġpower fully +Ġper l +ĠSh ade +ĠQu art +ĠE EG +Ġfisher man +OS ED +ĠTyp ical +df x +Ġmes hes +Ġet ched +worth iness +Ġtopp led +Ġ3 96 +or ius +We iss +Ġmy sql +ĠVal halla +Ù Ĵ +le asing +Ġrec omp +rap nel +S el +04 3 +Ġder ailed +ĠGu ides +IR T +Ġde human +ĠBritt any +" )) +Ġex claim +Ġb alk +Ġ8 40 +CLA IM +int el +L AB +Ġpe gged +Ġast roph +sm oking +Ġrig ging +Ġfix ation +Ġcat apult +ins ide +ĠC ascade +ĠBolshe vik +G aza +Dep th +Ġloud spe +Ġalmond s +me yer +l eness +j en +f resh +Ġunbeat en +ĠSqu id +ĠPres umably +Tim er +B W +Ġro sters +Ġell ipt +ĠHar riet +dat abase +ĠMut ual +ĠComm odore +uk ed +kn ife +ĠCOMM UN +h ya +Ġmel ts +arch ives +Ġrat ification +Ġmultip lying +Ġinter oper +Ġasc ert +w ings +ver ting +ĠScorp ion +ay e +ĠPorts mouth +ĠM TA +n it +iaz ep +Ġqu arantine +Ġslides how +Ġcent imeters +Ġsyn opsis +Ġsp ate +th irst +Ġnom inating +ĠMel vin +Pre view +Ġthro b +Ġgener ational +ĠRad ius +rest ling +put able +aw ar +N ECT +Ġunlaw fully +ĠRevel ations +Wik ipedia +sur v +Ġeye ing +ij n +ĠF W +Ġbr unt +Ġinter stellar +Ġcl itor +ĠCroat ian +ĠCh ic +ev a +ĠDis app +ĠA kin +iner ies +d ust +Interest ed +Ġgen esis +ĠE ucl +ö n +p icking +Ġmut ated +Ġdisappro ve +ĠHD L +Ġ6 25 +Ì ¶ +c ancer +Ġsqu ats +Ġle vers +Disc uss += ] +D ex +ĠVIDE OS +A UD +Ġtrans act +ĠKin ect +ĠK uala +ĠC yp +7 47 +Ġsh attering +Ġarsen ic +ĠInt ake +ĠAngel o +ĠQu it +ĠK he +Ġ18 93 +M aker +0 29 +ĠPain ting +Dis able +9 16 +Ġanal ges +Ġtact ile +Ġprop hes +Ġd iced +ĠTravel s +ĠHe ader +ĠClub s +Ass istant +Ġinc rim +Ġd ips +Ġcruc ifix +ĠShan ahan +ĠInter pret +Ġ40 90 +al ogy +abb a +Ġsimul ac +hus band +S IM +Ġrecy cle +uc er +ed ged +Ġre naissance +ĠBomb ay +Cath olic +ĠL INE +ĠCl othing +re ports +Ġpl aus +Ġd ag +ĠM ace +Z I +Ġintr uder +ĠVeter inary +g ru +Ġsne aky +ĠS ie +ĠC innamon +P OSE +Ġcou rier +ĠC NS +Ġemanc ipation +s it +Ġplay through +ĠFac ilities +v irt +ĠG auntlet +Thom pson +Ġunbeliev ably +Param eters +Ġst itching +ign e +ĠTH ESE +Priv acy +Ġshenan igans +Ġvit ri +ĠVal id +59 1 +Ń · +ĠProt otype +ink a +SC P +ĠT id +è Ī +old ed +Ġindividual ity +Ġbark ing +Ġm ars +ĠW D +Ġ8 20 +Ġt ir +Ġsl apping +Ġdisgr untled +ĠAng ola +ri us +ĠTorn ado +ĠTh urs +Ġcapt cha +Ġang st +ĠP og +ĠAssass ins +ĠAd idas +Ġjoy ful +Ġwh ining +Emer gency +Ġphosph orus +Ġatt rition +oph on +ĠTimber wolves +ĠJ ah +ĠBr inging +ĠW ad +ĠEn sure +oh l +ĠX ie +omm el +c mp +Ġz ipper +Ġrel at +ĠCor ridor +m ilo +T ING +Av g +Ġcro pped +] } +Ġr aged +ĠLump ur +ĠGuer rero +our ke +N ut +Ġoff sets +og lu +dr m +Ġmort als +lat able +Ġdismiss ive +ä¸ ī +Ġthro ats +Ġchips et +ĠSpot light +Catal og +art ist +G b +Ġch illy +Ġst oked +Ġ3 74 +W ard +L atin +Ġf iasco +Ġble ach +Ġb rav +Enh anced +Ġin oc +ĠFior ina +_ > +Ġle ukemia +Ġel uc +Ġannoun cer +ĠLith uan +ĠArm ageddon +å ĩ +Len in +ĠR uk +Ġpe pp +ĠRom antic +ĠP IT +ĠInter stellar +ĠAt kinson +R aid +J s +Go al +C ourse +Ġvan ishing +es ley +ĠR ounds +Els a +59 3 +Ġredund ancy +ĠST AND +Ġprop hetic +Ġhabit able +ry u +Ġfaint ly +M ODE +Ġfl anked +IR C +Aw esome +Ġsp urious +ĠZ ah +ĠMS G +Ġsh ading +Ġmotiv ational +ĠSant ana +ĠS PR +Ġexc ruciating +om ial +ĠM iko +ĠLe opard +A byss +Ġ[ | +d irty +Ġbath s +Ġdem oral +and re +P B +Ġun ification +Ġsac rament +Ġ[ & +Ġpric eless +Ġgel atin +Ġeman ating +ĠAll aah +98 6 +Ġout burst +Ġer as +ĠX VI +ĠSP I +O tt +ĠLaz arus +PL IED +F lying +blog s +W isconsin +R aven +Ġreb ate +Ġcreep s +ĠSp an +ĠPain ter +ĠKir a +ĠAm os +ĠCor vette +Cons umer +ĠRec over +ck i +Ġpes ky +ĠIn vention +Compan ies +Ġchalleng ers +ad emic +ĠUkrain ians +ĠNeuro log +ĠFors aken +Ġent rants +Ġemb attled +Ġdef unct +ĠGlac ier +Ġpo isons +ĠH orses +m akes +ĠD irt +Ġ4 23 +hh h +ĠTrans formation +QUI RE +................ .. +Ġtrave ller +ĠSe xy +ĠK ern +ip olar +Ġransom ware +oooooooo oooooooo +E c +rub y +Prof essional +ĠOut break +arg ument +G rey +ĠFif a +ĠCH O +ĠFOR M +ĠAm trak +- [ +Ġcr adle +Ġantioxid ants +ãģ®å ® +7 36 +ĠNAS L +ĠContribut ions +Ind iana +ĠST EP +C SS +Ġsal ient +Ġall ocations +yr ights +Ġm ashed +ĠCut ter +Sex ual +Ġp ounded +Ġfan base +Ġc asc +ĠTrans parency +Ġanaly tic +ĠSummon er +× ŀ +ĠAD C +det ail +Ġvan quished +Ġcr abs +ar ie +Dest roy +ĠS ack +Ġtrans istor +Al abama +ĠK oen +ĠFisher ies +c one +Ġannex ed +ĠM GM +es a +Ġf aked +ĠCong ratulations +Ġhind ered +Ġcorrection al +ĠI TV +lee ve +Ġin appropriately +lic ks +Ġtresp ass +Ġp aws +Ġnegoti ator +ĠChrist ensen +lim its +ĠDian ne +Ġeleg ance +ĠContract s +an ke +Ob j +Ġvigil ance +Ġcast les +ĠN AD +ĠHol o +Ġemph atically +ĠTit us +ĠServ ing +ĠRich ie +ĠP igs +5 68 +Ġanim osity +ĠAtt ributes +ĠU riel +M Q +my ra +ĠApplic ant +Ġpsychiat rists +ĠV ij +ĠAb by +ag ree +P ush +Ġk Wh +hib a +Ġinc ite +ĠWe asley +ĠTax i +minist ic +hy per +ĠF arn +Ġ6 01 +ĠNation wide +F ake +95 2 +Ġma ize +Ġinteract ed +Ġtransition ed +Ġparas itic +Ġharm onic +Ġdec aying +Ġbas eless +ns ics +Ġtrans pired +Ġabund antly +ĠFore nsic +Ġtread mill +ĠJ av +ab and +Ġssh d +Ġfront man +ĠJak arta +oll er +dro ps +ĠSERV ICES +rompt u +oph ical +h ospital +bled on +6 45 +Ġmid range +ĠEV ENT +cul ated +raw led +Ġper ched +Ġover board +ĠPe el +ĠP wr +ĠCar th +ĠCOM PLE +co e +sh all +Ġdeter rence +M ETHOD +ĠAbs ent +M EN +Ġs ill +ĠLE VEL +Y ork +Ġsin ners +ĠOP EC +ĠN ur +ĠDesign s +se lection +Ġunw orthy +CH A +Ġstreng thens +88 3 +ed ly +Ġslic ing +Ġmal nutrition +Ġfilm making +ĠPol k +ur ated +Ġ4 21 +bre akers +!' " +Ġwet lands +ĠDisc rimination +Ġallow able +Ġste ered +ĠSic ily +S AM +Ġmust ache +Ġm ids +Ġcl ipped +Ġcirc ulate +Ġbr ittle +ĠBuild ings +ra ised +ĠRound up +Ġwealth ier +Ġoverw rite +Ġover powered +ĠGerr ard +s ites +PD ATED +Ġacute ly +ĠGam ble +Ġp im +ĠK us +Typ ically +De ploy +ĠMoroc can +p otion +com be +Ġvigil ante +Ġ36 3 +St ew +ĠB agg +Ġres ided +ĠSp o +Ġrem nant +Ġempt iness +br ainer +Ġout patient +pri ority +Ġle ptin +ĠPay ton +ĠGle aming +ĠS hed +ĠPol o +ĠMormon ism +rest ricted +arl ane +w x +Ġcreat ine +ĠAn on +ĠST UD +ĠJ UL +ĠT ee +5 28 +08 9 +Ġhat ched +Dis patch +ĠCompos ite +Ġ45 1 +p uff +ĠX COM +ĠOr n +ĠTH ANK +END ED +ĠAshe ville +Ġà ľ +Ġman go +ĠS lightly +world ly +ĠW ander +ĠExp and +ĠCh r +M ist +Ġorthodox y +ĠUN ESCO +reg ate +Else where +k ie +ir led +Ġtopp le +Ġadopt ive +ĠLeg s +d ress +ĠS agan +b are +ĠGl ou +Cr unch +Ġhelp ers +Ġchron ically +ĠH uma +1 0000 +Ġaccommod ating +äº Ķ +Ġwrink les +Ġdod ged +four th +Ġpre con +Ġcompress or +ĠK are +Ġev ict +ĠWar wick +im ar +Ġmodern ization +Ġband wagon +Ġref uted +Ġnet ted +ĠNa ples +ĠGen ie +per ors +Ġfield ed +Ġde re +ĠPar ables +le es +Ġtr out +asp ers +Ġn ihil +Ġhapp iest +Ġflo ppy +ĠLo ft +ĠHe ard +Ġun ison +Ġl ug +ĠRed mond +class ic +Supp orters +SH IP +G MT +Ġfue lled +ç IJ +Ġd d +ĠEmin em +Ġ18 97 +NY SE +Ġsecret aries +ĠF IA +ĠCanaver al +F avorite +Ġp omp +Ġdetain ee +ers hip +aim on +i our +ĠA pex +Ġplant ations +am ia +ac ion +R ust +Ġtow ed +ĠTru ly +5 77 +Ġshel tered +r ider +W o +Ġl air +ĠInt elligent +impro ve +m atically +Ġet iquette +ad ra +all o +ĠJun o +any thing +ĠStru ggle +ĠPred ict +ĠGr imes +ĠAMER ICA +ct x +ĠSit uation +W OOD +Ġsol uble +me ier +Ġintoler able +ang ering +Ġun interrupted +Ġtool tip +Ġinterrog ated +Ġgun ned +ĠSne ak +æŃ ¦ +Ġt ether +Ġcr umble +L ens +Ġclust ered +ĠSy l +ĠHas an +Ġdystop ian +w ana +Ġjoy stick +ĠTh ib +amm u +Tom orrow +5 46 +Ġoverc ame +Ġminim ized +cept or +Run ner +ENG TH +ĠBrend a +ĠAchieve ments +Ġtor ches +Ġrapp ort +ĠInvestig ator +ĠHand ling +rel ation +g rey +8 15 +Ġk cal +ĠComm ands +d q +Ġcur ls +Ġbe arer +Ġcyn icism +it ri +ĠUse ful +B ee +D CS +Ġab ras +P ract +BIL ITIES +7 12 +Ġdebug ger +Ġdebt or +ĠL ia +ĠK ers +Ġexacerb ate +ĠSt acy +ĠB land +ĠSc enes +Ġbranch ing +âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ +ape ake +Ġs alsa +Ġmish and +ĠKon ami +ĠN ib +Ġanecd ote +Ġagree able +Ï ī +ĠNath aniel +ĠHe isman +ĠB eware +Ġ18 86 +spect ive +69 1 +5 22 +Ġinhib its +Ġhas hing +Ġ18 89 +å° Ĩ +v ich +P ure +Ġsolid ly +Ġaspir in +im aru +Ġstreet car +ĠU CS +ĠJ udd +Ġflash backs +p ins +Ġ14 40 +ĠUN HCR +ĠSym ptoms +T IT +5 38 +F ra +% ); +Ġo oz +Ġcur few +Ġcal med +Ġparticip ates +Te X +Ġnons ensical +Ġfull back +ĠDe L +mon key +h ari +Ġmetabol ites +Ġloot ed +ĠAL WAYS +ĠB CC +L t +oc het +B one +Ġveto ed +Ġg cc +ĠCL ICK +Ġ18 88 +s af +Ġstiff ness +Ġlow ly +ĠGe h +vers on +ors et +Ġun foreseen +Ġan esthesia +ĠOpt ical +Ġrecon structed +ĠT up +sh ows +NEW S +ĠNewsp aper +ĠA SA +ter a +N umbers +Ġinexpl icable +× ij +Ġhard ness +unt arily +ĠA cer +grad ient +ARD IS +Ġwood land +Ġmetaph ors +ĠWem bley +ĠPa vel +phil is +Ġre writing +Ġpercept ual +Ġ10 70 +worm s +ĠDown s +Ġunsur prisingly +Ġtag ging +fl ame +Ġlit res +Ġboun ces +ĠB abe +sh ut +Ġoverd oses +ĠShe ila +ĠCh au +ĠBl ess +Capt ure +ĠSign ificant +ĠSc ion +Ġ38 9 +ĠMc H +ĠTitan ium +ĠMe al +amed a +ag ents +agg ressive +B illy +76 3 +ĠS aying +DER R +it one +Coll ins +B ound +Ġbol ted +ĠDM CA +95 3 +Ġun iqueness +Ġep igen +un ci +ant am +Ġreck oning +ch airs +OG R +ĠSen egal +Ġ18 62 +re levant +Ġ ¯ +Ġpharm acies +ĠG eral +v ier +Y an +OR PG +Ġrab id +b ending +ĠUN ITED +Ġ4 65 +As sembly +Ġwe ep +Ġbe hest +ĠMother s +ĠJ ace +h id +Ġwh irlwind +ĠUN IVERS +Ġut opian +Ġkidn ap +Ph ilipp +K in +89 3 +Ġlivest ream +ĠM ISS +Ġsub versive +ĠTechn iques +ĠJUST ICE +ĠB ASE +Ġ38 7 +Ġassail ants +ĠHard core +Ġsprink led +ĠP se +é ļ +print ed +ĠH au +OR GE +ĠT OUR +Ġl aced +Ġit ch +G iving +Ġport ed +78 1 +//////////////// //////////////// +bre eding +Ġlog ger +ĠH OL +inn ie +First ly +Ġembry onic +Ġdeleg ated +p ai +O IL +Ġcentr ally +ĠR x +ĠSc outing +D utch +Ġhe reditary +ĠCru iser +s at +5 29 +ĠMar riott +other mal +Ġprohib itions +E arn +ĠSt ab +ĠColleg es +ĠBel ief +st retched +ĠL H +ĠEntity Item +C IA +Ġun rem +Ġlaure ate +Ġdenomin ations +sum mary +h ler +S pect +ĠK laus +ĠBe ans +Ġins ur +ĠPA X +Ġfield er +ĠV et +ĠSp arrow +z ie +ĠS Q +ĠMond ays +ĠOff line +ĠLer ner +ĠExt ensions +Ire land +Ġpatron age +Ġcontrast ed +ĠMan ia +h irt +Mos cow +Ġcondem ns +ĠAn ge +Ġcomp osing +ĠPe pe +ĠP addock +Ġheter ogeneity +Ġide ologically +Ġf ishes +Ġcur sing +ĠR utherford +ĠFlo ating +ĠAm elia +Te a +Syn opsis +Ġstun ts +Ġbe ad +Ġstock ing +ĠM ILL +ob ook +mass ive +\ < +Ġh ump +ĠPref erences +Engine Debug +ge ist +ĠNiet o +ome ver +ish y +eval uate +col onial +Altern ative +ĠGo Pro +ĠV ortex +ĠNET WORK +ans ky +Sec ure +ĠTh rust +Sn ake +Ġparcel s +Ġsam urai +Ġactress es +N ap +M F +ifer ation +Be er +5 23 +ĠI ly +oint ment +P ing +Ġstri ped +ĠMell on +oss ession +Ġneut ron +end ium +Ġa ph +ĠFlav oring +Ġ38 3 +Ġrespons iveness +ĠJ indal +ĠHitch cock +Den ver +ĠDRAG ON +sm anship +ĠDu pl +Ġs ly +Ġweb cam +ĠTw ain +ĠDar ling +ili ate +cons umer +D IT +Ġnames ake +Ġun orthodox +Ġfun er +ĠPL oS +ĠCONTR OL +ozy g +ogl obin +F ACE +ER G +ĠD ia +ĠF iesta +ce le +0 34 +Ġencl ave +âĸ¬ âĸ¬ +on ement +al ist +M and +Ġhome grown +ĠF ancy +Ġconcept ions +ĠCont ains +ure en +Ġreiter ate +Ġme ager +Ġinstall ments +Sp awn +6 27 +Ġphot oc +ĠCab rera +ĠRos enthal +ĠLans ing +is ner +Ġinvest s +ĠUFO s +EX P +Hard ware +Ġtr agically +Ġconced es +ie ft +ch am +bor gh +ĠSch r +ĠMel anie +ĠH oy +Ġvisit ation +Ġid iosyncr +Ġfract ions +Ġfore skin +ob os +Ġpo aching +ĠVI EW +Ġstimul ates +ĠG ork +can on +M IC +ĠNem esis +ĠInd ra +ĠDM V +Ġ5 29 +Ġinspect ing +Ġgrand ma +ĠW hedon +ĠSh ant +ĠP urg +ik an +ĠT eg +ĠCL R +z ac +Vict oria +ĠVer ify +ion ics +Ġpart ying +ĠM ou +col our +Ġtestim onies +l ations +Ġpress uring +hi ro +ac ers +Ġf id +ang ler +ĠCS I +Ġhere after +Ġdiss idents +report ing +iph any +che v +Ġsol itude +Ġl obe +Ġind is +Ġcred ential +re cent +ad ult +ĠNir vana +ĠFranch ise +L ayer +H yp +ĠBerks hire +Ġwill s +t if +Ġtot em +ĠJud ah +rep air +Inst ant +5 48 +Ġemb assies +Ġbott leneck +Ġb ount +Ġtyp ew +ĠAl vin +j ing +im ilar +R ush +Ġbr im +ĠHEL P +A im +] ' +Ġpass ively +Ġbound ed +ĠR ated +Ġcriminal ity +Ġbiom ark +Ġdisp atcher +ĠTow ards +Ġ+ ++ +right eous +f rog +ĠP anc +C arter +0 32 +æ© Ł +Ġult raviolet +ĠLic ensed +ĠT ata +ĠBl essing +ĠG AM +Ġchem ically +ĠSe af +ĠRE LE +ĠMerc enary +capital ist +Ġform ulations +Ġann ihilation +ĠVer b +ĠAr gon +Ġun loaded +Ġmorp hed +Ġconqu ering +back er +I ELD +Ġtheft s +Ġfront runner +ĠRoy ale +ĠFund amental +el ight +C hip +necess ary +ay n +ĠSl ip +Ġ4 48 +cern ed +P ause +Ġshock ingly +ĠAB V +Ġcomp osure +7 33 +ĠMotors port +ah ime +Mur ray +M ach +Ġgr ids +Ġdeb ian +Ġfurther more +Ġdexter ity +ĠCollect ions +os lov +il age +b j +ĠMont eneg +Ġstrut Connector +Ġmassac res +Ġbrief s +fet ched +uv ian +ol ition +Fail ure +emon ic +Ġfl ared +Ġclaim ant +Ġc ures +Ġgive aways +ĠSubst ance +al ions +Ġcr inge +ĠK ul +Ġarist ocracy +ĠUl ster +ol ated +h ousing +ĠM IS +Ġgl ared +ĠWil helm +ne eds +lam bda +build ers +ĠV IS +Ġradi ator +ĠGhost busters +Ġ4 36 +act ual +Ġher ds +ç a +watch ing +Ġcounter ing +Ch arge +Ġchar red +Ġwar heads +Ġiod ine +ĠM acy +04 1 +Ġdepart ures +ĠS ins +Ġdy ed +ĠConcept s +g ado +7 13 +Ġquot ations +Ġg ist +ĠChrist y +Ġant igen +ĠHem p +ĠD rawn +ĠB arg +ez vous +Ġp aternity +Ġar du +ĠAnch orage +ĠR ik +Ġover loaded +ĠUs ername +ĠTam my +ĠN au +ĠCell ular +Ġw aning +Ġrod ent +ĠWor cester +il ts +ĠT ad +Ġdwell ings +Ġbull ish +4 31 +Ġretali ate +Ġmig raine +ĠChev ron +CH ECK +Ġdon key +c rim +SP A +ĠAn alog +Ġmarqu ee +ĠHa as +B ir +ĠGD DR +ĠDownload s +Ġwill power +ĠFor th +ĠRecord ed +Ġimp ossibility +ĠLog ged +ĠFr anks +ĠR att +in itions +Ġclean ers +Ġsore ly +Ġflick ering +ĠEx amination +c atching +allow een +Ms g +Ġdun no +F a +Ġdys ph +c razy +.' '. +Ġmain line +Ġc s +Ġp tr +ĠW ally +ig un +95 1 +ĠBig foot +f ights +Ġretrie ving +J r +Ġdupl ication +ĠExpl an +Ġrel ational +Ġqu aint +Ġbisc uits +Ġad o +Ġsh udder +Ġantid ote +blood ed +ks h +Ġsa uces +Ġrein vest +Ġdispens ary +ĠD iver +Ġ9 000 +stud ent +Ġin separ +esc ap +Ġtodd lers +ĠGP IO +ĠAss ignment +head ers +Ġlack luster +Ġab ack +95 6 +Ġtool bar +7 45 +Ġo ust +Ġcontempl ation +ĠPRES IDENT +Ġ4 58 +==== == +Ġguarantee ing +ĠHe ist +ĠCann es +Ļ ½ +Ġcollabor ator +ĠAm p +Ġg ou +ĠSH ALL +st ories +78 3 +Ġmobil ized +Ġbro od +ĠL U +ĠðŁ ij +Ġref in +ĠAnthrop ology +v ind +ill i +Ġwarrant ies +ĠB abel +Ġsw ath +Ġc aches +Ġantagon ists +art ifacts +Ġhot ly +ĠSt arts +ĠG ö +z ag +!! !!! +Ġsc ourge +Ġcons piring +ru its +re verse +ĠShe en +ĠJes uit +ĠGiov anni +ad ies +Ġbutt ocks +ear cher +ac an +Ġvolley ball +Ġshroud ed +Ġscore board +b ats +ĠI PM +Ġass es +Ġde regulation +ĠTe legram +ĠReb oot +Ġ7 000 +ĠCan ary +Ġk ernels +ĠFranç ois +ĠD uff +ĠP on +ĠLe ica +ĠGar min +Ġor phans +ĠClaud ia +Ġcal endars +ĠLe ilan +ent o +R ocket +Ġbr unch +ĠHaw king +ain ers +Ġsens ibilities +Ġk W +ĠK and +Ġre claimed +Ġinteresting ly +× © +rom y +J M +ĠEnhance ment +b ush +Sk ip +Ġrapp ers +Ġg azing +p edia +ath lon +Rev olution +Ġsn ipers +Ġre verted +Ġconglomer ate +T erry +79 4 +Ġhars her +Ġdes olate +ĠHit man +Comm ission +Ġ( / +â̦ ." +Com par +Ġampl ification +om inated +Ġreg ress +ĠColl ider +Ġinform ants +Ġg azed diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/torch/megatron/megatron-gpt-345m_vocab b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/torch/megatron/megatron-gpt-345m_vocab new file mode 100644 index 00000000..1f1d9aac --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/torch/megatron/megatron-gpt-345m_vocab @@ -0,0 +1 @@ +{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "\u00a1": 94, "\u00a2": 95, "\u00a3": 96, "\u00a4": 97, "\u00a5": 98, "\u00a6": 99, "\u00a7": 100, "\u00a8": 101, "\u00a9": 102, "\u00aa": 103, "\u00ab": 104, "\u00ac": 105, "\u00ae": 106, "\u00af": 107, "\u00b0": 108, "\u00b1": 109, "\u00b2": 110, "\u00b3": 111, "\u00b4": 112, "\u00b5": 113, "\u00b6": 114, "\u00b7": 115, "\u00b8": 116, "\u00b9": 117, "\u00ba": 118, "\u00bb": 119, "\u00bc": 120, "\u00bd": 121, "\u00be": 122, "\u00bf": 123, "\u00c0": 124, "\u00c1": 125, "\u00c2": 126, "\u00c3": 127, "\u00c4": 128, "\u00c5": 129, "\u00c6": 130, "\u00c7": 131, "\u00c8": 132, "\u00c9": 133, "\u00ca": 134, "\u00cb": 135, "\u00cc": 136, "\u00cd": 137, "\u00ce": 138, "\u00cf": 139, "\u00d0": 140, "\u00d1": 141, "\u00d2": 142, "\u00d3": 143, "\u00d4": 144, "\u00d5": 145, "\u00d6": 146, "\u00d7": 147, "\u00d8": 148, "\u00d9": 149, "\u00da": 150, "\u00db": 151, "\u00dc": 152, "\u00dd": 153, "\u00de": 154, "\u00df": 155, "\u00e0": 156, "\u00e1": 157, "\u00e2": 158, "\u00e3": 159, "\u00e4": 160, "\u00e5": 161, "\u00e6": 162, "\u00e7": 163, "\u00e8": 164, "\u00e9": 165, "\u00ea": 166, "\u00eb": 167, "\u00ec": 168, "\u00ed": 169, "\u00ee": 170, "\u00ef": 171, "\u00f0": 172, "\u00f1": 173, "\u00f2": 174, "\u00f3": 175, "\u00f4": 176, "\u00f5": 177, "\u00f6": 178, "\u00f7": 179, "\u00f8": 180, "\u00f9": 181, "\u00fa": 182, "\u00fb": 183, "\u00fc": 184, "\u00fd": 185, "\u00fe": 186, "\u00ff": 187, "\u0100": 188, "\u0101": 189, "\u0102": 190, "\u0103": 191, "\u0104": 192, "\u0105": 193, "\u0106": 194, "\u0107": 195, "\u0108": 196, "\u0109": 197, "\u010a": 198, "\u010b": 199, "\u010c": 200, "\u010d": 201, "\u010e": 202, "\u010f": 203, "\u0110": 204, "\u0111": 205, "\u0112": 206, "\u0113": 207, "\u0114": 208, "\u0115": 209, "\u0116": 210, "\u0117": 211, "\u0118": 212, "\u0119": 213, "\u011a": 214, "\u011b": 215, "\u011c": 216, "\u011d": 217, "\u011e": 218, "\u011f": 219, "\u0120": 220, "\u0121": 221, "\u0122": 222, "\u0123": 223, "\u0124": 224, "\u0125": 225, "\u0126": 226, "\u0127": 227, "\u0128": 228, "\u0129": 229, "\u012a": 230, "\u012b": 231, "\u012c": 232, "\u012d": 233, "\u012e": 234, "\u012f": 235, "\u0130": 236, "\u0131": 237, "\u0132": 238, "\u0133": 239, "\u0134": 240, "\u0135": 241, "\u0136": 242, "\u0137": 243, "\u0138": 244, "\u0139": 245, "\u013a": 246, "\u013b": 247, "\u013c": 248, "\u013d": 249, "\u013e": 250, "\u013f": 251, "\u0140": 252, "\u0141": 253, "\u0142": 254, "\u0143": 255, "\u0120t": 256, "\u0120a": 257, "he": 258, "in": 259, "re": 260, "on": 261, "\u0120the": 262, "er": 263, "\u0120s": 264, "at": 265, "\u0120w": 266, "\u0120o": 267, "en": 268, "\u0120c": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "\u0120b": 275, "ed": 276, "\u0120f": 277, "ing": 278, "\u0120p": 279, "ou": 280, "\u0120an": 281, "al": 282, "ar": 283, "\u0120to": 284, "\u0120m": 285, "\u0120of": 286, "\u0120in": 287, "\u0120d": 288, "\u0120h": 289, "\u0120and": 290, "ic": 291, "as": 292, "le": 293, "\u0120th": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "\u0120n": 299, "\u0120l": 300, "st": 301, "\u0120re": 302, "ve": 303, "\u0120e": 304, "ro": 305, "ly": 306, "\u0120be": 307, "\u0120g": 308, "\u0120T": 309, "ct": 310, "\u0120S": 311, "id": 312, "ot": 313, "\u0120I": 314, "ut": 315, "et": 316, "\u0120A": 317, "\u0120is": 318, "\u0120on": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "\u0120that": 326, "\u0120C": 327, "ig": 328, "\u0120for": 329, "ac": 330, "\u0120y": 331, "ver": 332, "ur": 333, "\u0120u": 334, "ld": 335, "\u0120st": 336, "\u0120M": 337, "'s": 338, "\u0120he": 339, "\u0120it": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "\u0120you": 345, "il": 346, "\u0120B": 347, "\u0120wh": 348, "ol": 349, "\u0120P": 350, "\u0120with": 351, "\u01201": 352, "ter": 353, "ch": 354, "\u0120as": 355, "\u0120we": 356, "\u0120(": 357, "nd": 358, "ill": 359, "\u0120D": 360, "if": 361, "\u01202": 362, "ag": 363, "ers": 364, "ke": 365, "\u0120\"": 366, "\u0120H": 367, "em": 368, "\u0120con": 369, "\u0120W": 370, "\u0120R": 371, "her": 372, "\u0120was": 373, "\u0120r": 374, "od": 375, "\u0120F": 376, "ul": 377, "ate": 378, "\u0120at": 379, "ri": 380, "pp": 381, "ore": 382, "\u0120The": 383, "\u0120se": 384, "us": 385, "\u0120pro": 386, "\u0120ha": 387, "um": 388, "\u0120are": 389, "\u0120de": 390, "ain": 391, "and": 392, "\u0120or": 393, "igh": 394, "est": 395, "ist": 396, "ab": 397, "rom": 398, "\u0120N": 399, "th": 400, "\u0120com": 401, "\u0120G": 402, "un": 403, "op": 404, "00": 405, "\u0120L": 406, "\u0120not": 407, "ess": 408, "\u0120ex": 409, "\u0120v": 410, "res": 411, "\u0120E": 412, "ew": 413, "ity": 414, "ant": 415, "\u0120by": 416, "el": 417, "os": 418, "ort": 419, "oc": 420, "qu": 421, "\u0120from": 422, "\u0120have": 423, "\u0120su": 424, "ive": 425, "ould": 426, "\u0120sh": 427, "\u0120this": 428, "nt": 429, "ra": 430, "pe": 431, "ight": 432, "art": 433, "ment": 434, "\u0120al": 435, "ust": 436, "end": 437, "--": 438, "all": 439, "\u0120O": 440, "ack": 441, "\u0120ch": 442, "\u0120le": 443, "ies": 444, "red": 445, "ard": 446, "\u00e2\u0122": 447, "out": 448, "\u0120J": 449, "\u0120ab": 450, "ear": 451, "iv": 452, "ally": 453, "our": 454, "ost": 455, "gh": 456, "pt": 457, "\u0120pl": 458, "ast": 459, "\u0120can": 460, "ak": 461, "ome": 462, "ud": 463, "The": 464, "\u0120his": 465, "\u0120do": 466, "\u0120go": 467, "\u0120has": 468, "ge": 469, "'t": 470, "\u0120U": 471, "rou": 472, "\u0120sa": 473, "\u0120j": 474, "\u0120but": 475, "\u0120wor": 476, "\u0120all": 477, "ect": 478, "\u0120k": 479, "ame": 480, "\u0120will": 481, "ok": 482, "\u0120whe": 483, "\u0120they": 484, "ide": 485, "01": 486, "ff": 487, "ich": 488, "pl": 489, "ther": 490, "\u0120tr": 491, "..": 492, "\u0120int": 493, "ie": 494, "ure": 495, "age": 496, "\u0120ne": 497, "ial": 498, "ap": 499, "ine": 500, "ice": 501, "\u0120me": 502, "\u0120out": 503, "ans": 504, "one": 505, "ong": 506, "ions": 507, "\u0120who": 508, "\u0120K": 509, "\u0120up": 510, "\u0120their": 511, "\u0120ad": 512, "\u01203": 513, "\u0120us": 514, "ated": 515, "ous": 516, "\u0120more": 517, "ue": 518, "og": 519, "\u0120St": 520, "ind": 521, "ike": 522, "\u0120so": 523, "ime": 524, "per": 525, ".\"": 526, "ber": 527, "iz": 528, "act": 529, "\u0120one": 530, "\u0120said": 531, "\u0120-": 532, "are": 533, "\u0120your": 534, "cc": 535, "\u0120Th": 536, "\u0120cl": 537, "ep": 538, "ake": 539, "able": 540, "ip": 541, "\u0120cont": 542, "\u0120which": 543, "ia": 544, "\u0120im": 545, "\u0120about": 546, "\u0120were": 547, "very": 548, "ub": 549, "\u0120had": 550, "\u0120en": 551, "\u0120comp": 552, ",\"": 553, "\u0120In": 554, "\u0120un": 555, "\u0120ag": 556, "ire": 557, "ace": 558, "au": 559, "ary": 560, "\u0120would": 561, "ass": 562, "ry": 563, "\u0120\u00e2\u0122": 564, "cl": 565, "ook": 566, "ere": 567, "so": 568, "\u0120V": 569, "ign": 570, "ib": 571, "\u0120off": 572, "\u0120te": 573, "ven": 574, "\u0120Y": 575, "ile": 576, "ose": 577, "ite": 578, "orm": 579, "\u0120201": 580, "\u0120res": 581, "\u0120man": 582, "\u0120per": 583, "\u0120other": 584, "ord": 585, "ult": 586, "\u0120been": 587, "\u0120like": 588, "ase": 589, "ance": 590, "ks": 591, "ays": 592, "own": 593, "ence": 594, "\u0120dis": 595, "ction": 596, "\u0120any": 597, "\u0120app": 598, "\u0120sp": 599, "int": 600, "ress": 601, "ations": 602, "ail": 603, "\u01204": 604, "ical": 605, "\u0120them": 606, "\u0120her": 607, "ount": 608, "\u0120Ch": 609, "\u0120ar": 610, "\u0120if": 611, "\u0120there": 612, "\u0120pe": 613, "\u0120year": 614, "av": 615, "\u0120my": 616, "\u0120some": 617, "\u0120when": 618, "ough": 619, "ach": 620, "\u0120than": 621, "ru": 622, "ond": 623, "ick": 624, "\u0120over": 625, "vel": 626, "\u0120qu": 627, "\u010a\u010a": 628, "\u0120sc": 629, "reat": 630, "ree": 631, "\u0120It": 632, "ound": 633, "port": 634, "\u0120also": 635, "\u0120part": 636, "fter": 637, "\u0120kn": 638, "\u0120bec": 639, "\u0120time": 640, "ens": 641, "\u01205": 642, "ople": 643, "\u0120what": 644, "\u0120no": 645, "du": 646, "mer": 647, "ang": 648, "\u0120new": 649, "----": 650, "\u0120get": 651, "ory": 652, "ition": 653, "ings": 654, "\u0120just": 655, "\u0120into": 656, "\u01200": 657, "ents": 658, "ove": 659, "te": 660, "\u0120people": 661, "\u0120pre": 662, "\u0120its": 663, "\u0120rec": 664, "\u0120tw": 665, "ian": 666, "irst": 667, "ark": 668, "ors": 669, "\u0120work": 670, "ade": 671, "ob": 672, "\u0120she": 673, "\u0120our": 674, "wn": 675, "ink": 676, "lic": 677, "\u012019": 678, "\u0120He": 679, "ish": 680, "nder": 681, "ause": 682, "\u0120him": 683, "ons": 684, "\u0120[": 685, "\u0120ro": 686, "form": 687, "ild": 688, "ates": 689, "vers": 690, "\u0120only": 691, "oll": 692, "\u0120spe": 693, "ck": 694, "ell": 695, "amp": 696, "\u0120acc": 697, "\u0120bl": 698, "ious": 699, "urn": 700, "ft": 701, "ood": 702, "\u0120how": 703, "hed": 704, "\u0120'": 705, "\u0120after": 706, "aw": 707, "\u0120att": 708, "ov": 709, "ne": 710, "\u0120play": 711, "erv": 712, "ict": 713, "\u0120could": 714, "itt": 715, "\u0120am": 716, "\u0120first": 717, "\u01206": 718, "\u0120act": 719, "\u0120$": 720, "ec": 721, "hing": 722, "ual": 723, "ull": 724, "\u0120comm": 725, "oy": 726, "old": 727, "ces": 728, "ater": 729, "\u0120fe": 730, "\u0120bet": 731, "we": 732, "iff": 733, "\u0120two": 734, "ock": 735, "\u0120back": 736, ").": 737, "ident": 738, "\u0120under": 739, "rough": 740, "sel": 741, "xt": 742, "\u0120may": 743, "round": 744, "\u0120po": 745, "ph": 746, "iss": 747, "\u0120des": 748, "\u0120most": 749, "\u0120did": 750, "\u0120add": 751, "ject": 752, "\u0120inc": 753, "fore": 754, "\u0120pol": 755, "ont": 756, "\u0120again": 757, "clud": 758, "tern": 759, "\u0120know": 760, "\u0120need": 761, "\u0120cons": 762, "\u0120co": 763, "\u0120.": 764, "\u0120want": 765, "\u0120see": 766, "\u01207": 767, "ning": 768, "iew": 769, "\u0120This": 770, "ced": 771, "\u0120even": 772, "\u0120ind": 773, "ty": 774, "\u0120We": 775, "ath": 776, "\u0120these": 777, "\u0120pr": 778, "\u0120use": 779, "\u0120because": 780, "\u0120fl": 781, "ng": 782, "\u0120now": 783, "\u0120\u00e2\u0122\u0135": 784, "com": 785, "ise": 786, "\u0120make": 787, "\u0120then": 788, "ower": 789, "\u0120every": 790, "\u0120Un": 791, "\u0120sec": 792, "oss": 793, "uch": 794, "\u0120em": 795, "\u0120=": 796, "\u0120Re": 797, "ied": 798, "rit": 799, "\u0120inv": 800, "lect": 801, "\u0120supp": 802, "ating": 803, "\u0120look": 804, "man": 805, "pect": 806, "\u01208": 807, "row": 808, "\u0120bu": 809, "\u0120where": 810, "ific": 811, "\u0120years": 812, "ily": 813, "\u0120diff": 814, "\u0120should": 815, "\u0120rem": 816, "Th": 817, "In": 818, "\u0120ev": 819, "day": 820, "'re": 821, "rib": 822, "\u0120rel": 823, "ss": 824, "\u0120def": 825, "\u0120right": 826, "\u0120sy": 827, "),": 828, "les": 829, "000": 830, "hen": 831, "\u0120through": 832, "\u0120Tr": 833, "__": 834, "\u0120way": 835, "\u0120don": 836, "\u0120,": 837, "\u012010": 838, "ased": 839, "\u0120ass": 840, "ublic": 841, "\u0120reg": 842, "\u0120And": 843, "ix": 844, "\u0120very": 845, "\u0120includ": 846, "other": 847, "\u0120imp": 848, "oth": 849, "\u0120sub": 850, "\u0120\u00e2\u0122\u0136": 851, "\u0120being": 852, "arg": 853, "\u0120Wh": 854, "==": 855, "ible": 856, "\u0120does": 857, "ange": 858, "ram": 859, "\u01209": 860, "ert": 861, "ps": 862, "ited": 863, "ational": 864, "\u0120br": 865, "\u0120down": 866, "\u0120many": 867, "aking": 868, "\u0120call": 869, "uring": 870, "ities": 871, "\u0120ph": 872, "ics": 873, "als": 874, "\u0120dec": 875, "ative": 876, "ener": 877, "\u0120before": 878, "ility": 879, "\u0120well": 880, "\u0120much": 881, "erson": 882, "\u0120those": 883, "\u0120such": 884, "\u0120ke": 885, "\u0120end": 886, "\u0120But": 887, "ason": 888, "ting": 889, "\u0120long": 890, "ef": 891, "\u0120think": 892, "ys": 893, "\u0120bel": 894, "\u0120sm": 895, "its": 896, "ax": 897, "\u0120own": 898, "\u0120prov": 899, "\u0120set": 900, "ife": 901, "ments": 902, "ble": 903, "ward": 904, "\u0120show": 905, "\u0120pres": 906, "ms": 907, "omet": 908, "\u0120ob": 909, "\u0120say": 910, "\u0120Sh": 911, "ts": 912, "ful": 913, "\u0120eff": 914, "\u0120gu": 915, "\u0120inst": 916, "und": 917, "ren": 918, "cess": 919, "\u0120ent": 920, "\u0120You": 921, "\u0120good": 922, "\u0120start": 923, "ince": 924, "\u0120made": 925, "tt": 926, "stem": 927, "olog": 928, "up": 929, "\u0120|": 930, "ump": 931, "\u0120hel": 932, "vern": 933, "ular": 934, "ually": 935, "\u0120ac": 936, "\u0120mon": 937, "\u0120last": 938, "\u0120200": 939, "10": 940, "\u0120stud": 941, "ures": 942, "\u0120Ar": 943, "self": 944, "ars": 945, "meric": 946, "ues": 947, "cy": 948, "\u0120min": 949, "ollow": 950, "\u0120col": 951, "io": 952, "\u0120mod": 953, "\u0120count": 954, "\u0120Com": 955, "hes": 956, "\u0120fin": 957, "air": 958, "ier": 959, "\u00e2\u0122\u0136": 960, "read": 961, "ank": 962, "atch": 963, "ever": 964, "\u0120str": 965, "\u0120point": 966, "ork": 967, "\u0120New": 968, "\u0120sur": 969, "ool": 970, "alk": 971, "ement": 972, "\u0120used": 973, "ract": 974, "ween": 975, "\u0120same": 976, "oun": 977, "\u0120Al": 978, "ci": 979, "\u0120differe": 980, "\u0120while": 981, "--------": 982, "\u0120game": 983, "cept": 984, "\u0120sim": 985, "...": 986, "\u0120inter": 987, "ek": 988, "\u0120report": 989, "\u0120produ": 990, "\u0120still": 991, "led": 992, "ah": 993, "\u0120here": 994, "\u0120world": 995, "\u0120though": 996, "\u0120num": 997, "arch": 998, "imes": 999, "ale": 1000, "\u0120Se": 1001, "\u0120If": 1002, "//": 1003, "\u0120Le": 1004, "\u0120ret": 1005, "\u0120ref": 1006, "\u0120trans": 1007, "ner": 1008, "ution": 1009, "ters": 1010, "\u0120take": 1011, "\u0120Cl": 1012, "\u0120conf": 1013, "way": 1014, "ave": 1015, "\u0120going": 1016, "\u0120sl": 1017, "ug": 1018, "\u0120Americ": 1019, "\u0120spec": 1020, "\u0120hand": 1021, "\u0120between": 1022, "ists": 1023, "\u0120De": 1024, "oot": 1025, "It": 1026, "\u0120ear": 1027, "\u0120against": 1028, "\u0120high": 1029, "gan": 1030, "az": 1031, "ather": 1032, "\u0120exp": 1033, "\u0120op": 1034, "\u0120ins": 1035, "\u0120gr": 1036, "\u0120help": 1037, "\u0120requ": 1038, "ets": 1039, "ins": 1040, "\u0120Pro": 1041, "ism": 1042, "\u0120found": 1043, "land": 1044, "ata": 1045, "uss": 1046, "ames": 1047, "\u0120person": 1048, "\u0120great": 1049, "pr": 1050, "\u0120sign": 1051, "\u0120An": 1052, "'ve": 1053, "\u0120somet": 1054, "\u0120ser": 1055, "hip": 1056, "\u0120run": 1057, "\u0120:": 1058, "\u0120ter": 1059, "irect": 1060, "\u0120follow": 1061, "\u0120det": 1062, "ices": 1063, "\u0120find": 1064, "12": 1065, "\u0120mem": 1066, "\u0120cr": 1067, "ered": 1068, "ex": 1069, "\u0120ext": 1070, "uth": 1071, "ense": 1072, "co": 1073, "\u0120team": 1074, "ving": 1075, "ouse": 1076, "ash": 1077, "att": 1078, "ved": 1079, "\u0120system": 1080, "\u0120As": 1081, "der": 1082, "ives": 1083, "min": 1084, "\u0120lead": 1085, "\u0120Bl": 1086, "cent": 1087, "\u0120around": 1088, "\u0120govern": 1089, "\u0120cur": 1090, "velop": 1091, "any": 1092, "\u0120cour": 1093, "alth": 1094, "ages": 1095, "ize": 1096, "\u0120car": 1097, "ode": 1098, "\u0120law": 1099, "\u0120read": 1100, "'m": 1101, "con": 1102, "\u0120real": 1103, "\u0120support": 1104, "\u012012": 1105, "....": 1106, "\u0120really": 1107, "ness": 1108, "\u0120fact": 1109, "\u0120day": 1110, "\u0120both": 1111, "ying": 1112, "\u0120serv": 1113, "\u0120For": 1114, "\u0120three": 1115, "\u0120wom": 1116, "\u0120med": 1117, "ody": 1118, "\u0120They": 1119, "50": 1120, "\u0120exper": 1121, "ton": 1122, "\u0120each": 1123, "akes": 1124, "\u0120che": 1125, "\u0120cre": 1126, "ines": 1127, "\u0120rep": 1128, "19": 1129, "gg": 1130, "illion": 1131, "\u0120grou": 1132, "ute": 1133, "ik": 1134, "We": 1135, "get": 1136, "ER": 1137, "\u0120met": 1138, "\u0120says": 1139, "ox": 1140, "\u0120during": 1141, "ern": 1142, "ized": 1143, "ared": 1144, "\u0120fam": 1145, "ically": 1146, "\u0120happ": 1147, "\u0120Is": 1148, "\u0120char": 1149, "med": 1150, "vent": 1151, "\u0120gener": 1152, "ient": 1153, "ple": 1154, "iet": 1155, "rent": 1156, "11": 1157, "ves": 1158, "ption": 1159, "\u012020": 1160, "formation": 1161, "\u0120cor": 1162, "\u0120offic": 1163, "ield": 1164, "\u0120too": 1165, "ision": 1166, "\u0120inf": 1167, "\u0120Z": 1168, "the": 1169, "oad": 1170, "\u0120public": 1171, "\u0120prog": 1172, "ric": 1173, "**": 1174, "\u0120war": 1175, "\u0120power": 1176, "view": 1177, "\u0120few": 1178, "\u0120loc": 1179, "\u0120different": 1180, "\u0120state": 1181, "\u0120head": 1182, "'ll": 1183, "\u0120poss": 1184, "\u0120stat": 1185, "ret": 1186, "ants": 1187, "\u0120val": 1188, "\u0120iss": 1189, "\u0120cle": 1190, "ivers": 1191, "anc": 1192, "\u0120expl": 1193, "\u0120another": 1194, "\u0120Q": 1195, "\u0120av": 1196, "thing": 1197, "nce": 1198, "Wh": 1199, "\u0120child": 1200, "\u0120since": 1201, "ired": 1202, "less": 1203, "\u0120life": 1204, "\u0120develop": 1205, "ittle": 1206, "\u0120dep": 1207, "\u0120pass": 1208, "\u00e3\u0125": 1209, "\u0120turn": 1210, "orn": 1211, "This": 1212, "bers": 1213, "ross": 1214, "\u0120Ad": 1215, "\u0120fr": 1216, "\u0120resp": 1217, "\u0120second": 1218, "oh": 1219, "\u0120/": 1220, "\u0120disc": 1221, "\u0120&": 1222, "\u0120something": 1223, "\u0120comple": 1224, "\u0120ed": 1225, "\u0120fil": 1226, "\u0120month": 1227, "aj": 1228, "uc": 1229, "\u0120government": 1230, "\u0120without": 1231, "\u0120leg": 1232, "\u0120dist": 1233, "\u0120put": 1234, "\u0120quest": 1235, "ann": 1236, "\u0120prot": 1237, "20": 1238, "\u0120never": 1239, "ience": 1240, "\u0120level": 1241, "\u0120art": 1242, "\u0120things": 1243, "\u0120might": 1244, "\u0120effect": 1245, "\u0120contro": 1246, "\u0120cent": 1247, "\u012018": 1248, "\u0120allow": 1249, "\u0120belie": 1250, "chool": 1251, "ott": 1252, "\u0120incre": 1253, "\u0120feel": 1254, "\u0120result": 1255, "\u0120lot": 1256, "\u0120fun": 1257, "ote": 1258, "\u0120ty": 1259, "erest": 1260, "\u0120contin": 1261, "\u0120using": 1262, "\u0120big": 1263, "201": 1264, "\u0120ask": 1265, "\u0120best": 1266, "\u0120)": 1267, "IN": 1268, "\u0120opp": 1269, "30": 1270, "\u0120number": 1271, "iness": 1272, "St": 1273, "lease": 1274, "\u0120ca": 1275, "\u0120must": 1276, "\u0120direct": 1277, "\u0120gl": 1278, "\u0120<": 1279, "\u0120open": 1280, "\u0120post": 1281, "\u0120come": 1282, "\u0120seem": 1283, "ording": 1284, "\u0120week": 1285, "ately": 1286, "ital": 1287, "\u0120el": 1288, "riend": 1289, "\u0120far": 1290, "\u0120tra": 1291, "inal": 1292, "\u0120pri": 1293, "\u0120US": 1294, "\u0120place": 1295, "\u0120form": 1296, "\u0120told": 1297, "\":": 1298, "ains": 1299, "ature": 1300, "\u0120Trump": 1301, "\u0120stand": 1302, "\u0120#": 1303, "ider": 1304, "\u0120Fr": 1305, "\u0120next": 1306, "\u0120soc": 1307, "\u0120pur": 1308, "\u0120let": 1309, "\u0120little": 1310, "\u0120hum": 1311, "\u0120i": 1312, "ron": 1313, "15": 1314, "\u012015": 1315, "\u0120commun": 1316, "\u0120mark": 1317, "\u0120There": 1318, "\u0120wr": 1319, "\u0120That": 1320, "\u0120information": 1321, "ways": 1322, "\u0120bus": 1323, "app": 1324, "\u0120invest": 1325, "me": 1326, "\u0120hard": 1327, "ained": 1328, "ead": 1329, "\u0120import": 1330, "\u0120appro": 1331, "\u0120test": 1332, "\u0120tri": 1333, "\u0120rest": 1334, "osed": 1335, "\u0120full": 1336, "\u0120care": 1337, "\u0120Sp": 1338, "\u0120case": 1339, "ON": 1340, "\u0120sk": 1341, "\u0120less": 1342, "\u0120+": 1343, "\u0120partic": 1344, "\u0120Pl": 1345, "ably": 1346, "uck": 1347, "ished": 1348, "chn": 1349, "be": 1350, "\u0120list": 1351, "ator": 1352, "\u0120top": 1353, "\u0120adv": 1354, "\u0120Be": 1355, "ruct": 1356, "\u0120dem": 1357, "ration": 1358, "ling": 1359, "gy": 1360, "reen": 1361, "ger": 1362, "\u0120home": 1363, "\u0120left": 1364, "\u0120better": 1365, "\u0120data": 1366, "\u012011": 1367, "\u0120attack": 1368, "\u0120proble": 1369, "line": 1370, "ards": 1371, "\u0120beh": 1372, "ral": 1373, "\u0120How": 1374, "\u0120She": 1375, "arge": 1376, "\u0120--": 1377, "://": 1378, "\u0120bro": 1379, "\u0120Ph": 1380, "ats": 1381, "\u0120build": 1382, "ww": 1383, "ided": 1384, "aim": 1385, "ases": 1386, "ency": 1387, "\u0120main": 1388, "ined": 1389, "\u0120including": 1390, "\u0120{": 1391, "\u0120got": 1392, "\u0120interest": 1393, "\u0120keep": 1394, "\u0120X": 1395, "\u0120eas": 1396, "aining": 1397, "\u0120class": 1398, "\u00e2\u0122\u00a6": 1399, "\u0120No": 1400, "\u0120var": 1401, "\u0120small": 1402, "ample": 1403, "AT": 1404, "\u0120ide": 1405, "\u0120So": 1406, "\u0120rece": 1407, "\u0120polit": 1408, "\u0120mov": 1409, "\u0120plan": 1410, "\u0120percent": 1411, "iving": 1412, "\u0120camp": 1413, "\u0120pay": 1414, "14": 1415, "sc": 1416, "ised": 1417, "\u0120unt": 1418, "oney": 1419, "ploy": 1420, "====": 1421, "\u0120didn": 1422, "\u0120Ind": 1423, "els": 1424, "ertain": 1425, "\u0120pos": 1426, "____": 1427, "iver": 1428, "\u0120process": 1429, "\u0120program": 1430, "ified": 1431, "\u0120Rep": 1432, "16": 1433, "uro": 1434, "ology": 1435, "atter": 1436, "ina": 1437, "\u0120name": 1438, "\u0120All": 1439, "\u0120four": 1440, "\u0120return": 1441, "vious": 1442, "bs": 1443, "\u0120called": 1444, "\u0120move": 1445, "\u0120Sc": 1446, "ird": 1447, "\u0120group": 1448, "\u0120bre": 1449, "\u0120men": 1450, "\u0120cap": 1451, "ten": 1452, "ee": 1453, "\u0120dri": 1454, "leg": 1455, "here": 1456, "uthor": 1457, "\u0120pat": 1458, "\u0120current": 1459, "ides": 1460, "\u0120pop": 1461, "to": 1462, "ention": 1463, "\u0120always": 1464, "\u0120mil": 1465, "\u0120women": 1466, "\u012016": 1467, "\u0120old": 1468, "iven": 1469, "raph": 1470, "\u0120Or": 1471, "ror": 1472, "ently": 1473, "\u0120near": 1474, "\u0120Ex": 1475, "ream": 1476, "sh": 1477, "\u012014": 1478, "\u0120free": 1479, "ission": 1480, "stand": 1481, "\u0120Con": 1482, "ality": 1483, "used": 1484, "13": 1485, "\u0120design": 1486, "\u0120change": 1487, "\u0120chang": 1488, "\u0120bo": 1489, "\u0120vis": 1490, "ember": 1491, "\u0120book": 1492, "ready": 1493, "\u0120kill": 1494, "25": 1495, "pped": 1496, "\u0120away": 1497, "\u0120able": 1498, "\u0120country": 1499, "\u0120const": 1500, "arn": 1501, "\u0120order": 1502, "AR": 1503, "ior": 1504, "ium": 1505, "orth": 1506, "18": 1507, "ailable": 1508, "\u0120sw": 1509, "\u0120million": 1510, "\u012013": 1511, "atic": 1512, "ted": 1513, "\u0120Go": 1514, "\u0120oper": 1515, "eng": 1516, "\u0120thing": 1517, "ajor": 1518, "conom": 1519, "\u0120Comm": 1520, "\u0120why": 1521, "ured": 1522, "ural": 1523, "\u0120school": 1524, "by": 1525, "\u0120Mar": 1526, "\u0120aff": 1527, "\u0120days": 1528, "\u0120ann": 1529, "ush": 1530, "ane": 1531, "If": 1532, "eg": 1533, "\u0120prof": 1534, "\u0120health": 1535, "outh": 1536, "But": 1537, "ional": 1538, ".,": 1539, "\u0120sol": 1540, "\u0120already": 1541, "\u012030": 1542, "\u0120charact": 1543, "He": 1544, "\u0120friend": 1545, "ES": 1546, "ians": 1547, "icle": 1548, "'d": 1549, "\u0120On": 1550, "\u0120least": 1551, "\u0120prom": 1552, "\u0120dr": 1553, "\u0120hist": 1554, "ither": 1555, "\u0120est": 1556, "iqu": 1557, "17": 1558, "son": 1559, "\u0120tell": 1560, "\u0120talk": 1561, "ohn": 1562, "oint": 1563, "lection": 1564, "AN": 1565, "\u0120until": 1566, "augh": 1567, "\u0120later": 1568, "\u0120ve": 1569, "\u0120view": 1570, "ending": 1571, "ived": 1572, "\u0120word": 1573, "ware": 1574, "\u0120cost": 1575, "\u0120enough": 1576, "\u0120give": 1577, "\u0120United": 1578, "\u0120techn": 1579, "arent": 1580, "OR": 1581, "\u0120par": 1582, "\u0120Dr": 1583, "\u01202016": 1584, "rist": 1585, "ering": 1586, "\u0120\u00c2": 1587, "\u0120large": 1588, "side": 1589, "acy": 1590, "ccess": 1591, "\u0120win": 1592, "\u0120important": 1593, "\u0120199": 1594, "\u0120doesn": 1595, "\u012017": 1596, "\u0120business": 1597, "\u0120clear": 1598, "\u0120rese": 1599, "\",": 1600, "ury": 1601, "\u0120equ": 1602, "aster": 1603, "alf": 1604, "\u0120American": 1605, "nect": 1606, "\u0120expect": 1607, "iversity": 1608, "\u0120occ": 1609, "\u0120Fl": 1610, "\u0120kind": 1611, "\u0120mean": 1612, "\u0120past": 1613, "\u0120dev": 1614, "\u0120bas": 1615, "let": 1616, "raft": 1617, "\u0120organ": 1618, "\u0120del": 1619, "\u0120perform": 1620, "\u0120story": 1621, "\u0120season": 1622, "\u0120Col": 1623, "\u0120claim": 1624, "\u0120came": 1625, "\u0120within": 1626, "\u0120line": 1627, "\u0120project": 1628, "\u0120At": 1629, "\u0120control": 1630, "ended": 1631, "\u0120Sy": 1632, "\u0120air": 1633, "ization": 1634, "\u0120*": 1635, "ley": 1636, "\u0120money": 1637, "idd": 1638, "You": 1639, "for": 1640, "\u0120family": 1641, "\u0120making": 1642, "\u0120bit": 1643, "\u0120police": 1644, "\u0120happen": 1645, "\u0120vers": 1646, "ony": 1647, "uff": 1648, "\u0120When": 1649, "\u0120sit": 1650, "ideo": 1651, "lf": 1652, "ison": 1653, "\u0120sure": 1654, "gin": 1655, "\u0120appear": 1656, "\u0120light": 1657, "\u0120es": 1658, "of": 1659, "\u0120water": 1660, "\u0120times": 1661, "not": 1662, "\u0120grow": 1663, "\u0120company": 1664, "\u0120Te": 1665, "ows": 1666, "\u0120mar": 1667, "ource": 1668, "iol": 1669, "arm": 1670, "br": 1671, "\u0120example": 1672, "\u0120conc": 1673, "\u0120fore": 1674, "\u0120To": 1675, "pro": 1676, "EN": 1677, "ries": 1678, "\u012025": 1679, "\u0120Can": 1680, "ney": 1681, "\u0120actually": 1682, "\u0120ever": 1683, "urity": 1684, "aken": 1685, "aps": 1686, "\u0120tax": 1687, "\u0120major": 1688, "ama": 1689, "\u0120often": 1690, "eral": 1691, "\u0120human": 1692, "\u0120job": 1693, "ister": 1694, "\u0120available": 1695, "ocr": 1696, "enn": 1697, "aid": 1698, "ivid": 1699, "\u0120record": 1700, "?\"": 1701, "\u0120sing": 1702, "\u0120Am": 1703, "idence": 1704, "\u0120news": 1705, "ster": 1706, "\u0120econom": 1707, "\u0120following": 1708, "\u0120Br": 1709, "ising": 1710, "\u0120hour": 1711, "most": 1712, "ument": 1713, "\u0120sex": 1714, "\u0120desc": 1715, "\u0120become": 1716, "\u0120Ed": 1717, "\u0120took": 1718, "\u0120having": 1719, "\u0120product": 1720, "ault": 1721, "As": 1722, "aring": 1723, "\u0120means": 1724, "\u0120hop": 1725, "une": 1726, "\u0120cho": 1727, "\u0120certain": 1728, "\u0120non": 1729, "\u0120deal": 1730, "24": 1731, "lement": 1732, "oci": 1733, "ene": 1734, "\u0120side": 1735, "\u0120Pr": 1736, "\u0120May": 1737, "\u0120reason": 1738, "ued": 1739, "ched": 1740, "ulation": 1741, "\u0120elect": 1742, "\u0120official": 1743, "\u0120possible": 1744, "\u0120hold": 1745, "ands": 1746, "ots": 1747, "\u0120city": 1748, "ories": 1749, "\u0120sever": 1750, "\u0120children": 1751, "\u0120once": 1752, "\u0120activ": 1753, "ler": 1754, "\u0120night": 1755, "itions": 1756, "\u0120John": 1757, "ape": 1758, "play": 1759, "\u0120done": 1760, "\u0120lim": 1761, "\u0120working": 1762, "\u0120Pres": 1763, "orld": 1764, "eb": 1765, "\u0120Co": 1766, "\u0120body": 1767, "ails": 1768, "utes": 1769, "\u0120Mr": 1770, "\u0120whether": 1771, "\u0120author": 1772, "rop": 1773, "\u0120proper": 1774, "\u0120seen": 1775, ");": 1776, "\u0120fac": 1777, "\u0120Su": 1778, "\u0120cond": 1779, "iting": 1780, "\u0120course": 1781, "\u0120}": 1782, "----------------": 1783, "aign": 1784, "\u0120event": 1785, "\u0120eng": 1786, "\u0120pot": 1787, "\u0120intern": 1788, "iam": 1789, "\u0120short": 1790, "empt": 1791, "\u00e3\u0124": 1792, "\u0120God": 1793, "ilar": 1794, "80": 1795, "\u0120orig": 1796, "IS": 1797, "ourn": 1798, "ability": 1799, "itive": 1800, "\u0120dam": 1801, "\u0120100": 1802, "\u0120press": 1803, "\u0120doing": 1804, "\u0120protect": 1805, "ring": 1806, "\u0120thought": 1807, "\u0120question": 1808, "rew": 1809, "\u0120War": 1810, "\u0120several": 1811, "\u0120State": 1812, "\u0120given": 1813, "\u0120fund": 1814, "\u0120Tw": 1815, "\u0120went": 1816, "ances": 1817, "work": 1818, "por": 1819, "my": 1820, "40": 1821, "\u0120arg": 1822, "artment": 1823, "ustom": 1824, "\u0120polic": 1825, "\u0120meet": 1826, "\u0120creat": 1827, "22": 1828, "\u0120States": 1829, "\u0120games": 1830, "raw": 1831, "uture": 1832, "\u0120understand": 1833, "urs": 1834, "\u0120Ob": 1835, "lish": 1836, "sy": 1837, "\u0120makes": 1838, "\u0120won": 1839, "agon": 1840, "\u0120htt": 1841, "\u0120love": 1842, "ential": 1843, "\u0120complete": 1844, "par": 1845, "\u0120Im": 1846, "AL": 1847, "\u0120account": 1848, "\u00c2\u0142": 1849, "ored": 1850, "vert": 1851, "\u0120ident": 1852, "\u01202015": 1853, "\u0120others": 1854, "\u0120Min": 1855, "iber": 1856, "verage": 1857, "There": 1858, "itional": 1859, "dd": 1860, "\u0120prob": 1861, "\u0120young": 1862, "\u0120along": 1863, "\u0120according": 1864, "\u0120yet": 1865, "\u0120members": 1866, "\u0120What": 1867, "oid": 1868, "\u0120Man": 1869, "And": 1870, "\u0120among": 1871, "ai": 1872, "\u0120employ": 1873, "\u0120Res": 1874, "\u0120>": 1875, "\u0120invol": 1876, "\u0120low": 1877, "af": 1878, "\u0120Car": 1879, "\u0120hig": 1880, "\u0120One": 1881, "\u0120Sec": 1882, "ination": 1883, "\u0120likely": 1884, "\u0120ant": 1885, "aged": 1886, "\u0120Russ": 1887, "\u0120ben": 1888, "\u0120rele": 1889, "For": 1890, "back": 1891, "\u0120Not": 1892, "\u0120president": 1893, "ball": 1894, "\u0120access": 1895, "ividual": 1896, "\u0120Dem": 1897, "\u0120Euro": 1898, "60": 1899, "\u0120known": 1900, "irl": 1901, "\u0120Gr": 1902, "\u0120early": 1903, "use": 1904, "iety": 1905, "\u00e2\u0122\u0135": 1906, "\u0120fight": 1907, "\u0120sent": 1908, "\u0120today": 1909, "\u0120market": 1910, "\".": 1911, "\u0120based": 1912, "\u0120strong": 1913, "urther": 1914, "\u0120deb": 1915, "mber": 1916, "\u0120problem": 1917, "\u0120death": 1918, "\u0120social": 1919, "imate": 1920, "AS": 1921, "ortun": 1922, "\u0120campaign": 1923, "ery": 1924, "Ch": 1925, "\u0120ey": 1926, "ially": 1927, "\u0120mus": 1928, "wh": 1929, "pos": 1930, "\u0120er": 1931, "\u0120saf": 1932, "\u0120months": 1933, "iron": 1934, "\u0120viol": 1935, "\u0120five": 1936, "\u0120stre": 1937, "\u0120players": 1938, "inc": 1939, "ald": 1940, "year": 1941, "aun": 1942, "\u0120success": 1943, "\u0120present": 1944, "erence": 1945, "\u01202014": 1946, "\u0120sugg": 1947, "\u0120particular": 1948, "\u0120try": 1949, "\u0120suggest": 1950, "\u0120Christ": 1951, "ones": 1952, "\u0120priv": 1953, "23": 1954, "\u0120crit": 1955, "\u0120land": 1956, "\u0120local": 1957, "ify": 1958, "29": 1959, "\u0120aut": 1960, "ED": 1961, "\u0120Gu": 1962, "\u0120mult": 1963, "\u0120political": 1964, "\u0120asked": 1965, "\u0120former": 1966, "itter": 1967, "ript": 1968, "\u0120close": 1969, "\u0120pract": 1970, "\u0120York": 1971, "\u0120getting": 1972, "\u0120across": 1973, "\u0120comb": 1974, "\u0120believe": 1975, "\u0120z": 1976, "\u0120toget": 1977, "\u0120together": 1978, "\u0120Cent": 1979, "irc": 1980, "\u0120individual": 1981, "\u0120Mc": 1982, "27": 1983, "isk": 1984, "\u0120Eng": 1985, "\u0120face": 1986, "\u012024": 1987, "\u0120value": 1988, "\u0120area": 1989, "ev": 1990, "\u0120writ": 1991, "\u0120President": 1992, "\u0120vot": 1993, "\u0120key": 1994, "\u0120mom": 1995, "put": 1996, "\u0120anything": 1997, "\u0120experience": 1998, "attle": 1999, "\u0120mind": 2000, "aff": 2001, "omm": 2002, "\u0120future": 2003, "ged": 2004, "\u0120cut": 2005, "\u0120tot": 2006, "itch": 2007, "\u0120video": 2008, "\u0120investig": 2009, "\u0120net": 2010, "\u0120My": 2011, "rict": 2012, "ien": 2013, ".)": 2014, "\u0120impro": 2015, "though": 2016, "wards": 2017, "\u0120connect": 2018, "\u0120Med": 2019, "selves": 2020, "ensive": 2021, "mb": 2022, "ober": 2023, "ators": 2024, "An": 2025, "\u012050": 2026, "\u0120redu": 2027, "resent": 2028, "\u0120above": 2029, "\u0120fre": 2030, "\u0120Europe": 2031, "sw": 2032, "\u0120amount": 2033, "\u0120App": 2034, "\u0120either": 2035, "\u0120milit": 2036, "\u0120anal": 2037, "\u0120fail": 2038, "\u0120En": 2039, "ales": 2040, "\u0120special": 2041, "\u0120black": 2042, "IT": 2043, "cher": 2044, "\u0120looking": 2045, "\u0120fire": 2046, "yn": 2047, "\u0120almost": 2048, "oon": 2049, "\u0120study": 2050, "\u0120miss": 2051, "ches": 2052, "rown": 2053, "\u0120tre": 2054, "\u0120community": 2055, "\u0120media": 2056, "\u0120food": 2057, "\u0120comes": 2058, "\u0120University": 2059, "\u0120single": 2060, "What": 2061, "uly": 2062, "\u0120half": 2063, "ague": 2064, "hod": 2065, "\u0120Republic": 2066, "\u0120started": 2067, "\u0120quick": 2068, "oto": 2069, "book": 2070, "\u0120issue": 2071, "itor": 2072, "\u0120else": 2073, "\u0120consider": 2074, "26": 2075, "rodu": 2076, "\u0120taken": 2077, "28": 2078, "99": 2079, "\u0120With": 2080, "\u0120true": 2081, "\u0120wa": 2082, "\u0120trad": 2083, "\u0120ago": 2084, "\u0120mess": 2085, "ief": 2086, "\u0120added": 2087, "oke": 2088, "\u0120bad": 2089, "\u0120fav": 2090, "33": 2091, "\u0120similar": 2092, "ask": 2093, "\u0120Don": 2094, "\u0120character": 2095, "orts": 2096, "\u0120House": 2097, "\u0120reported": 2098, "\u0120type": 2099, "val": 2100, "iod": 2101, "\u0120However": 2102, "\u0120targ": 2103, "\u0120entire": 2104, "pping": 2105, "\u0120history": 2106, "\u0120live": 2107, "ffic": 2108, "........": 2109, "ederal": 2110, "\u0120trying": 2111, "\u0120discuss": 2112, "\u0120Har": 2113, "aces": 2114, "lished": 2115, "\u0120self": 2116, "osp": 2117, "rest": 2118, "\u0120room": 2119, "elt": 2120, "\u0120fall": 2121, "olution": 2122, "\u0120et": 2123, "\u0120x": 2124, "\u0120isn": 2125, "\u0120idea": 2126, "bo": 2127, "\u0120sound": 2128, "\u0120Dep": 2129, "\u0120someone": 2130, "cially": 2131, "ully": 2132, "\u0120foc": 2133, "\u0120object": 2134, "ift": 2135, "aper": 2136, "\u0120player": 2137, "\u0120rather": 2138, "\u0120service": 2139, "ashing": 2140, "\u0120Do": 2141, "\u0120Part": 2142, "rug": 2143, "mon": 2144, "ply": 2145, "\u0120mor": 2146, "\u0120nothing": 2147, "\u0120provide": 2148, "IC": 2149, "ung": 2150, "\u0120party": 2151, "\u0120exist": 2152, "\u0120mag": 2153, "70": 2154, "\u0120rul": 2155, "\u0120house": 2156, "\u0120behind": 2157, "\u0120however": 2158, "\u0120World": 2159, "\u0120sum": 2160, "\u0120applic": 2161, "\u0120;": 2162, "\u0120function": 2163, "gr": 2164, "\u0120Pol": 2165, "\u0120front": 2166, "200": 2167, "\u0120series": 2168, "\u0120tem": 2169, "\u0120typ": 2170, "ills": 2171, "\u0120opt": 2172, "\u0120points": 2173, "\u0120below": 2174, "itted": 2175, "\u0120specific": 2176, "\u01202017": 2177, "umb": 2178, "\u0120ra": 2179, "\u0120previous": 2180, "\u0120pret": 2181, "reme": 2182, "\u0120custom": 2183, "\u0120court": 2184, "\u0120Me": 2185, "\u0120repl": 2186, "\u0120whole": 2187, "go": 2188, "cer": 2189, "\u0120treat": 2190, "\u0120Act": 2191, "\u0120probably": 2192, "\u0120learn": 2193, "ender": 2194, "\u0120Ass": 2195, "\u0120version": 2196, "now": 2197, "\u0120check": 2198, "\u0120Cal": 2199, "RE": 2200, "minist": 2201, "On": 2202, "ources": 2203, "\u0120benef": 2204, "\u0120doc": 2205, "\u0120deter": 2206, "\u0120enc": 2207, "\u0120super": 2208, "\u0120address": 2209, "\u0120vict": 2210, "\u01202013": 2211, "\u0120meas": 2212, "tr": 2213, "\u0120field": 2214, "When": 2215, "\u0120signific": 2216, "uge": 2217, "\u0120feat": 2218, "\u0120common": 2219, "load": 2220, "\u0120begin": 2221, "\u0120bring": 2222, "\u0120action": 2223, "erman": 2224, "\u0120describ": 2225, "\u0120indust": 2226, "\u0120wanted": 2227, "ried": 2228, "ming": 2229, "\u0120attempt": 2230, "45": 2231, "fer": 2232, "\u0120due": 2233, "ression": 2234, "##": 2235, "\u0120shall": 2236, "\u0120six": 2237, "oo": 2238, "\u0120step": 2239, "\u0120pub": 2240, "\u0120himself": 2241, "\u012023": 2242, "\u0120cop": 2243, "\u0120dest": 2244, "\u0120stop": 2245, "AC": 2246, "ibility": 2247, "\u0120lab": 2248, "icult": 2249, "\u0120hours": 2250, "\u0120create": 2251, "\u0120further": 2252, "\u0120America": 2253, "\u0120City": 2254, "\u0120dou": 2255, "head": 2256, "ST": 2257, "\u0120North": 2258, "cing": 2259, "\u0120national": 2260, "ule": 2261, "\u0120Inst": 2262, "\u0120taking": 2263, "\u0120Qu": 2264, "irt": 2265, "\u0120red": 2266, "\u0120research": 2267, "viron": 2268, "\u0120Ge": 2269, "\u0120break": 2270, "ana": 2271, "\u0120space": 2272, "aterial": 2273, "\u0120recent": 2274, "\u0120Ab": 2275, "\u0120general": 2276, "\u0120hit": 2277, "\u0120period": 2278, "\u0120everything": 2279, "ively": 2280, "\u0120phys": 2281, "\u0120saying": 2282, "anks": 2283, "\u0120cou": 2284, "\u0120cult": 2285, "aced": 2286, "eal": 2287, "uation": 2288, "\u0120coun": 2289, "lu": 2290, "\u0120include": 2291, "\u0120position": 2292, "\u0120After": 2293, "\u0120Canad": 2294, "\u0120Em": 2295, "\u0120imm": 2296, "\u0120Red": 2297, "\u0120pick": 2298, "\u0120compl": 2299, "\u0120matter": 2300, "reg": 2301, "ext": 2302, "angu": 2303, "isc": 2304, "ole": 2305, "aut": 2306, "\u0120compet": 2307, "eed": 2308, "fect": 2309, "\u012021": 2310, "\u0120Sen": 2311, "\u0120These": 2312, "asing": 2313, "\u0120cannot": 2314, "\u0120init": 2315, "\u0120relations": 2316, "ached": 2317, "\u0120bar": 2318, "\u012040": 2319, "\u0120TH": 2320, "\u01202012": 2321, "\u0120vol": 2322, "\u0120ground": 2323, "\u0120security": 2324, "\u0120upd": 2325, "ilt": 2326, "35": 2327, "\u0120concern": 2328, "\u0120Just": 2329, "\u0120white": 2330, "\u0120seems": 2331, "\u0120Her": 2332, "pecially": 2333, "ients": 2334, "\u0120announ": 2335, "\u0120fig": 2336, "ights": 2337, "\u0120stri": 2338, "like": 2339, "ids": 2340, "\u0120sus": 2341, "\u0120watch": 2342, "\u0120\u00e2": 2343, "\u0120wind": 2344, "\u0120Cont": 2345, "\u0120itself": 2346, "\u0120mass": 2347, "Al": 2348, "yle": 2349, "ique": 2350, "\u0120National": 2351, "\u0120abs": 2352, "\u0120pack": 2353, "\u0120outside": 2354, "\u0120anim": 2355, "\u0120pain": 2356, "eter": 2357, "\u0120manag": 2358, "duct": 2359, "ogn": 2360, "\u0120]": 2361, "\u0120Sept": 2362, "sec": 2363, "off": 2364, "\u0120Jan": 2365, "\u0120foot": 2366, "ades": 2367, "\u0120third": 2368, "\u0120mot": 2369, "\u0120evidence": 2370, "inton": 2371, "\u0120threat": 2372, "apt": 2373, "ples": 2374, "cle": 2375, "\u0120lo": 2376, "\u0120decl": 2377, "\u0120item": 2378, "medi": 2379, "\u0120represent": 2380, "omb": 2381, "amer": 2382, "\u0120significant": 2383, "ograph": 2384, "su": 2385, "\u0120cal": 2386, "ires": 2387, "0000": 2388, "ID": 2389, "AM": 2390, "\u0120simply": 2391, "\u0120longer": 2392, "\u0120file": 2393, "OT": 2394, "che": 2395, "So": 2396, "ateg": 2397, "org": 2398, "\u0120His": 2399, "\u0120ener": 2400, "\u0120dom": 2401, "\u0120upon": 2402, "ili": 2403, "\":\"": 2404, "\u0120themselves": 2405, "\u0120coming": 2406, "\u0120quite": 2407, "\u0120difficult": 2408, "\u0120Bar": 2409, "ilities": 2410, "rel": 2411, "ends": 2412, "cial": 2413, "64": 2414, "\u0120woman": 2415, "rap": 2416, "yr": 2417, "\u0120necess": 2418, "ips": 2419, "\u0120text": 2420, "\u0120require": 2421, "\u0120military": 2422, "\u0120review": 2423, "\u0120respons": 2424, "75": 2425, "\u0120subject": 2426, "\u0120instead": 2427, "\u0120issues": 2428, "\u0120gen": 2429, "\",\"": 2430, "\u0120minutes": 2431, "\u0120weap": 2432, "ray": 2433, "amed": 2434, "time": 2435, "bl": 2436, "How": 2437, "\u0120code": 2438, "\u0120Sm": 2439, "\u0120higher": 2440, "\u0120Ste": 2441, "ris": 2442, "\u0120page": 2443, "\u0120students": 2444, "\u0120Intern": 2445, "\u0120method": 2446, "\u0120Aug": 2447, "\u0120Per": 2448, "\u0120Ag": 2449, "\u0120policy": 2450, "\u0120Sw": 2451, "\u0120exec": 2452, "\u0120accept": 2453, "ume": 2454, "ribut": 2455, "\u0120words": 2456, "\u0120final": 2457, "\u0120changes": 2458, "\u0120Democr": 2459, "\u0120friends": 2460, "\u0120respect": 2461, "\u0120ep": 2462, "\u0120compan": 2463, "ivil": 2464, "\u0120damage": 2465, "****": 2466, "ogle": 2467, "vironment": 2468, "\u0120neg": 2469, "ental": 2470, "\u0120ap": 2471, "\u0120total": 2472, "ival": 2473, "!\"": 2474, "lim": 2475, "\u0120needs": 2476, "\u0120agre": 2477, "\u0120development": 2478, "\u0120age": 2479, "iple": 2480, "21": 2481, "\u0120results": 2482, "\u0120Af": 2483, "Sh": 2484, "\u0120gun": 2485, "\u0120Obama": 2486, "roll": 2487, "\u0120@": 2488, "\u0120rights": 2489, "\u0120Brit": 2490, "\u0120running": 2491, "\u0120wasn": 2492, "\u0120port": 2493, "\u0120rate": 2494, "\u0120pretty": 2495, "\u0120target": 2496, "\u0120saw": 2497, "\u0120circ": 2498, "\u0120works": 2499, "icro": 2500, "alt": 2501, "over": 2502, "www": 2503, "That": 2504, "lier": 2505, "\u0120everyone": 2506, "ude": 2507, "\u0120pie": 2508, "iddle": 2509, "rael": 2510, "\u0120rad": 2511, "\u0120block": 2512, "\u0120walk": 2513, "To": 2514, "\u00e3\u0123": 2515, "nes": 2516, "\u0120Aust": 2517, "aul": 2518, "rote": 2519, "\u0120South": 2520, "ession": 2521, "oph": 2522, "\u0120shows": 2523, "\u0120site": 2524, "\u0120jo": 2525, "\u0120risk": 2526, "clus": 2527, "lt": 2528, "\u0120inj": 2529, "iding": 2530, "\u0120Spe": 2531, "\u0120chall": 2532, "irm": 2533, "\u012022": 2534, "itting": 2535, "str": 2536, "\u0120hy": 2537, "LE": 2538, "key": 2539, "\u0120began": 2540, "atur": 2541, "ashington": 2542, "lam": 2543, "\u0120Dav": 2544, "bit": 2545, "\u0120size": 2546, "\u0120Par": 2547, "38": 2548, "ournal": 2549, "face": 2550, "\u0120decision": 2551, "\u0120larg": 2552, "\u0120jud": 2553, "rect": 2554, "\u0120continue": 2555, "\u0120Oct": 2556, "overed": 2557, "\u0120Int": 2558, "========": 2559, "\u0120parent": 2560, "\u0120Will": 2561, "\u0120easy": 2562, "\u0120drug": 2563, "anger": 2564, "\u0120sense": 2565, "\u0120di": 2566, "iday": 2567, "\u0120energy": 2568, "istic": 2569, "\u0120associ": 2570, "arter": 2571, "obal": 2572, "eks": 2573, "\u0120El": 2574, "urch": 2575, "\u0120girl": 2576, "oe": 2577, "itle": 2578, "\u012028": 2579, "\u0120Che": 2580, "\u0120request": 2581, "\u0120soon": 2582, "\u0120host": 2583, "ky": 2584, "\u0120states": 2585, "omes": 2586, "\u0120material": 2587, "lex": 2588, "\u0120moment": 2589, "\u0120answ": 2590, "onse": 2591, "\u0120especially": 2592, "\u0120norm": 2593, "\u0120services": 2594, "pite": 2595, "ran": 2596, "\u0120role": 2597, "44": 2598, "):": 2599, "\u0120cred": 2600, "Cl": 2601, "________": 2602, "\u0120mat": 2603, "\u0120log": 2604, "\u0120Clinton": 2605, "OU": 2606, "\u0120office": 2607, "\u012026": 2608, "\u0120charg": 2609, "\u0120track": 2610, "ma": 2611, "\u0120heart": 2612, "\u0120ball": 2613, "\u0120personal": 2614, "\u0120building": 2615, "na": 2616, "set": 2617, "body": 2618, "\u0120Black": 2619, "\u0120increase": 2620, "itten": 2621, "\u0120needed": 2622, "36": 2623, "32": 2624, "=\"": 2625, "\u0120lost": 2626, "\u0120became": 2627, "\u0120groups": 2628, "\u0120Mus": 2629, "\u0120wrote": 2630, "\u0120Pe": 2631, "\u0120prop": 2632, "joy": 2633, "\u00c3\u00a9": 2634, "\u0120White": 2635, "\u0120dead": 2636, ".'": 2637, "\u0120http": 2638, "\u0120webs": 2639, "OS": 2640, "\u0120inside": 2641, "\u0120wrong": 2642, "\u0120statement": 2643, "\u0120...": 2644, "yl": 2645, "\u0120film": 2646, "\u0120music": 2647, "\u0120share": 2648, "ification": 2649, "\u0120release": 2650, "\u0120forward": 2651, "\u0120stay": 2652, "\u0120comput": 2653, "itte": 2654, "ser": 2655, "\u0120original": 2656, "\u0120card": 2657, "\u0120cand": 2658, "\u0120div": 2659, "atural": 2660, "\u0120favor": 2661, "OM": 2662, "\u0120cases": 2663, "uses": 2664, "\u0120section": 2665, "\u0120leave": 2666, "ging": 2667, "oved": 2668, "\u0120Washington": 2669, "39": 2670, "\u0120Gl": 2671, "\u0120required": 2672, "action": 2673, "apan": 2674, "oor": 2675, "iter": 2676, "\u0120King": 2677, "\u0120countries": 2678, "\u0120German": 2679, "lling": 2680, "\u012027": 2681, "34": 2682, "\u0120questions": 2683, "\u0120prim": 2684, "\u0120cell": 2685, "\u0120shoot": 2686, "\u0120anyone": 2687, "\u0120West": 2688, "\u0120affect": 2689, "epend": 2690, "\u0120online": 2691, "\u0120Israel": 2692, "\u0120September": 2693, "\u0120ability": 2694, "\u0120content": 2695, "ises": 2696, "\u0120reve": 2697, "\u0120laun": 2698, "\u0120indic": 2699, "\u0120force": 2700, "cast": 2701, "\u0120sold": 2702, "aving": 2703, "fl": 2704, "\u0120soft": 2705, "\u0120companies": 2706, "ceed": 2707, "\u0120article": 2708, "\u0120aud": 2709, "\u0120rev": 2710, "\u0120educ": 2711, "\u0120playing": 2712, "05": 2713, "\u0120held": 2714, "ctor": 2715, "\u0120released": 2716, "\u0120federal": 2717, "37": 2718, "\u0120administ": 2719, "\u0120interview": 2720, "\u0120install": 2721, "\u0120received": 2722, "\u0120source": 2723, "uk": 2724, "Ph": 2725, "\u0120serious": 2726, "\u0120created": 2727, "\u0120cause": 2728, "\u0120immedi": 2729, "\u0120defin": 2730, "uel": 2731, "\u0120Department": 2732, "ctions": 2733, "\u0120Cour": 2734, "\u0120Now": 2735, "ze": 2736, "ites": 2737, "itution": 2738, "\u0120late": 2739, "\u0120speak": 2740, "ners": 2741, "\u0120legal": 2742, "ari": 2743, "\u0120Cor": 2744, "\u0120weeks": 2745, "\u0120model": 2746, "\u0120pred": 2747, "\u0120exact": 2748, "BC": 2749, "\u0120By": 2750, "ING": 2751, "osing": 2752, "\u0120takes": 2753, "\u0120regard": 2754, "\u0120opportun": 2755, "\u0120price": 2756, "\u0120198": 2757, "\u0120Apr": 2758, "fully": 2759, "\u0120ord": 2760, "\u0120problems": 2761, "ruction": 2762, "ham": 2763, "\u0120Count": 2764, "lege": 2765, "\u0120leaders": 2766, "ET": 2767, "lev": 2768, "\u0120deep": 2769, "ological": 2770, "ese": 2771, "haps": 2772, "\u0120Some": 2773, "\u0120pers": 2774, "\u0120contract": 2775, "\u0120relationship": 2776, "sp": 2777, "oud": 2778, "\u0120base": 2779, "48": 2780, "mit": 2781, "Ad": 2782, "ancial": 2783, "\u0120consum": 2784, "\u0120potential": 2785, "\u0120langu": 2786, "rem": 2787, "eth": 2788, "\u0120relig": 2789, "ressed": 2790, "66": 2791, "\u0120link": 2792, "\u0120lower": 2793, "ayer": 2794, "\u0120June": 2795, "\u0120fem": 2796, "unt": 2797, "erc": 2798, "urd": 2799, "\u0120contact": 2800, "\u0120ill": 2801, "\u0120mother": 2802, "\u0120estab": 2803, "htt": 2804, "\u0120March": 2805, "\u0120Bro": 2806, "\u0120China": 2807, "\u012029": 2808, "\u0120squ": 2809, "\u0120provided": 2810, "\u0120average": 2811, "asons": 2812, "\u01202011": 2813, "\u0120exam": 2814, "lin": 2815, "55": 2816, "ned": 2817, "\u0120perfect": 2818, "\u0120tou": 2819, "alse": 2820, "ux": 2821, "\u0120buy": 2822, "\u0120shot": 2823, "\u0120collect": 2824, "\u0120phot": 2825, "\u0120played": 2826, "\u0120surpr": 2827, "\u0120officials": 2828, "\u0120simple": 2829, "avy": 2830, "\u0120industry": 2831, "\u0120hands": 2832, "ground": 2833, "\u0120pull": 2834, "\u0120round": 2835, "\u0120user": 2836, "\u0120range": 2837, "uary": 2838, "\u0120private": 2839, "ops": 2840, "ees": 2841, "\u0120ways": 2842, "\u0120Mich": 2843, "\u0120veh": 2844, "\u0120except": 2845, "\u0120terms": 2846, "imum": 2847, "pper": 2848, "ION": 2849, "ores": 2850, "\u0120Dragon": 2851, "oul": 2852, "\u0120den": 2853, "\u0120performance": 2854, "\u0120bill": 2855, "cil": 2856, "47": 2857, "\u0120environment": 2858, "\u0120exc": 2859, "add": 2860, "\u0120worth": 2861, "\u0120pict": 2862, "\u0120chance": 2863, "\u01202018": 2864, "bor": 2865, "\u0120speed": 2866, "iction": 2867, "\u0120alleg": 2868, "\u0120Japan": 2869, "atory": 2870, "reet": 2871, "\u0120match": 2872, "\u0120II": 2873, "\u0120stru": 2874, "order": 2875, "\u0120ste": 2876, "\u0120living": 2877, "\u0120struct": 2878, "ino": 2879, "\u0120separ": 2880, "hern": 2881, "\u0120response": 2882, "\u0120enjoy": 2883, "\u0120via": 2884, "AD": 2885, "uments": 2886, "acebook": 2887, "\u0120member": 2888, "ibr": 2889, "izing": 2890, "\u0120tool": 2891, "\u0120Mon": 2892, "\u0120While": 2893, "hood": 2894, "\u0120Ang": 2895, "\u0120Def": 2896, "\u0120offer": 2897, "Tr": 2898, "aur": 2899, "\u0120turned": 2900, "\u0120July": 2901, "down": 2902, "anced": 2903, "\u0120recently": 2904, "\u0120Ear": 2905, "\u0120ce": 2906, "\u0120Star": 2907, "\u0120Cong": 2908, "rought": 2909, "\u0120blood": 2910, "\u0120hope": 2911, "\u0120comment": 2912, "aint": 2913, "\u0120arri": 2914, "iles": 2915, "\u0120particip": 2916, "ought": 2917, "ription": 2918, "08": 2919, "49": 2920, "\u0120gave": 2921, "\u0120select": 2922, "\u0120killed": 2923, "sych": 2924, "\u0120goes": 2925, "ij": 2926, "\u0120coll": 2927, "\u0120impact": 2928, "atives": 2929, "\u0120Ser": 2930, "09": 2931, "\u0120August": 2932, "\u0120boy": 2933, "de": 2934, "\u0120Des": 2935, "\u0120felt": 2936, "US": 2937, "\u0120expected": 2938, "\u0120image": 2939, "\u0120Mark": 2940, "ccording": 2941, "oice": 2942, "EC": 2943, "\u0120Mag": 2944, "ened": 2945, "hold": 2946, "\u0120Post": 2947, "\u0120prevent": 2948, "No": 2949, "\u0120involved": 2950, "\u0120eyes": 2951, "\u0120quickly": 2952, "At": 2953, "unk": 2954, "\u0120behav": 2955, "\u0120ur": 2956, "\u0120led": 2957, "come": 2958, "ey": 2959, "\u0120candid": 2960, "\u0120earlier": 2961, "\u0120focus": 2962, "ety": 2963, "Pro": 2964, "ledge": 2965, "ixed": 2966, "illed": 2967, "\u0120popular": 2968, "AP": 2969, "\u0120sett": 2970, "light": 2971, "\u0120various": 2972, "inks": 2973, "\u0120levels": 2974, "\u0120road": 2975, "ellig": 2976, "ables": 2977, "hel": 2978, "ittee": 2979, "\u0120Gener": 2980, "ype": 2981, "\u0120heard": 2982, "icles": 2983, "\u0120mis": 2984, "\u0120users": 2985, "\u0120San": 2986, "\u0120improve": 2987, "\u0120father": 2988, "\u0120search": 2989, "They": 2990, "vil": 2991, "\u0120profess": 2992, "\u0120knew": 2993, "\u0120loss": 2994, "\u0120events": 2995, "65": 2996, "\u0120billion": 2997, "07": 2998, "02": 2999, "\u0120News": 3000, "\u0120AM": 3001, "\u0120cover": 3002, "where": 3003, "ension": 3004, "\u0120bott": 3005, "\u0120areas": 3006, "ences": 3007, "ope": 3008, "\u0120Twitter": 3009, "ael": 3010, "\u0120gets": 3011, "\u0120Google": 3012, "\u0120sn": 3013, "iant": 3014, "\u0120vote": 3015, "\u0120nearly": 3016, "\u0120included": 3017, "\u0120recogn": 3018, "zz": 3019, "mm": 3020, "aled": 3021, "\u0120happened": 3022, "04": 3023, "\u0120hot": 3024, "\u0120whose": 3025, "\u0120civil": 3026, "\u0120suff": 3027, "oes": 3028, "itiz": 3029, "\u0120Syri": 3030, "\u0120respond": 3031, "\u0120hon": 3032, "\u0120features": 3033, "\u0120economic": 3034, "\u0120April": 3035, "rim": 3036, "\u0120technology": 3037, "\u0120option": 3038, "aging": 3039, "\u0120purch": 3040, "Re": 3041, "\u0120lat": 3042, "chie": 3043, "isl": 3044, "\u0120recomm": 3045, "uf": 3046, "\u0120training": 3047, "\u0120effects": 3048, "\u0120fast": 3049, "\u01202010": 3050, "\u0120occur": 3051, "\u0120website": 3052, "\u0120email": 3053, "\u0120sens": 3054, "ech": 3055, "\u0120oil": 3056, "\u0120influ": 3057, "\u0120currently": 3058, "\u0120Sch": 3059, "\u0120Add": 3060, "\u0120goal": 3061, "\u0120scient": 3062, "\u0120conv": 3063, "100": 3064, "emy": 3065, "\u0120decided": 3066, "\u0120travel": 3067, "\u0120mention": 3068, "LL": 3069, "03": 3070, "\u0120election": 3071, "\u0120phone": 3072, "\u0120looks": 3073, "\u0120situation": 3074, "\u0120cy": 3075, "\u0120hor": 3076, "bed": 3077, "\u0120Court": 3078, "aily": 3079, "aves": 3080, "\u0120quality": 3081, "\u0120Comp": 3082, "wise": 3083, "\u0120table": 3084, "\u0120staff": 3085, "\u0120Wind": 3086, "ett": 3087, "\u0120tried": 3088, "idered": 3089, "\u0120addition": 3090, "\u0120box": 3091, "\u0120lack": 3092, "arily": 3093, "\u0120wide": 3094, "\u0120mid": 3095, "\u0120board": 3096, "ysis": 3097, "\u0120anti": 3098, "ha": 3099, "\u0120dig": 3100, "ening": 3101, "\u0120dro": 3102, "Con": 3103, "68": 3104, "\u0120slow": 3105, "based": 3106, "sequ": 3107, "\u0120path": 3108, "Ex": 3109, "aker": 3110, "\u0120worked": 3111, "\u0120pen": 3112, "\u0120engine": 3113, "\u0120looked": 3114, "\u0120Super": 3115, "\u0120Serv": 3116, "\u0120victim": 3117, "Un": 3118, "\u0120property": 3119, "\u0120introdu": 3120, "\u0120execut": 3121, "\u0120PM": 3122, "Le": 3123, "\u0120color": 3124, "\u0120More": 3125, "\u012060": 3126, "\u0120network": 3127, "\u0120date": 3128, "cul": 3129, "idge": 3130, "\u0120extra": 3131, "31": 3132, "\u0120sle": 3133, "67": 3134, "\u0120wond": 3135, "\u0120reports": 3136, "just": 3137, "\u0120Austral": 3138, "\u0120capital": 3139, "\u0120ens": 3140, "\u0120command": 3141, "\u0120allowed": 3142, "\u0120prep": 3143, "\u0120capt": 3144, "hib": 3145, "\u0120numbers": 3146, "chan": 3147, "\u0120fair": 3148, "mp": 3149, "oms": 3150, "\u0120reach": 3151, "With": 3152, "tain": 3153, "\u0120broad": 3154, "\u0120couple": 3155, "ecause": 3156, "lying": 3157, "\u0120Feb": 3158, "\u0120screen": 3159, "\u0120lives": 3160, "\u0120prior": 3161, "\u0120Congress": 3162, "Ar": 3163, "\u0120approach": 3164, "\u0120emer": 3165, "aries": 3166, "\u0120Dis": 3167, "serv": 3168, "\u0120Ne": 3169, "\u0120built": 3170, "cies": 3171, "\u0120repe": 3172, "\u0120rules": 3173, "force": 3174, "\u0120Pal": 3175, "\u0120financial": 3176, "\u0120considered": 3177, "\u0120Char": 3178, "nces": 3179, "\u0120IS": 3180, "\u0120brought": 3181, "\u0120bi": 3182, "iers": 3183, "\u0120Sim": 3184, "OP": 3185, "\u0120products": 3186, "\u0120visit": 3187, "\u0120document": 3188, "\u0120conduct": 3189, "\u0120completely": 3190, "ining": 3191, "\u0120Calif": 3192, "ibly": 3193, "\u0120written": 3194, "\u0120TV": 3195, "ements": 3196, "\u0120draw": 3197, "One": 3198, "\u0120published": 3199, "\u0120secret": 3200, "rain": 3201, "het": 3202, "\u0120Facebook": 3203, "onday": 3204, "\u0120Up": 3205, "\u0120sexual": 3206, "\u0120thous": 3207, "\u0120Pat": 3208, "\u0120ess": 3209, "\u0120standard": 3210, "\u0120arm": 3211, "ges": 3212, "ection": 3213, "\u0120fell": 3214, "\u0120foreign": 3215, "ani": 3216, "\u0120Friday": 3217, "\u0120regular": 3218, "inary": 3219, "\u0120increased": 3220, "\u0120usually": 3221, "\u0120demon": 3222, "\u0120dark": 3223, "\u0120additional": 3224, "rol": 3225, "\u0120Of": 3226, "\u0120production": 3227, "!!": 3228, "undred": 3229, "\u0120international": 3230, "idents": 3231, "\u0120Free": 3232, "roup": 3233, "\u0120race": 3234, "\u0120mach": 3235, "\u0120huge": 3236, "All": 3237, "lear": 3238, "ovember": 3239, "\u0120town": 3240, "\u0120attention": 3241, "\u0120Off": 3242, "yond": 3243, "\u0120Then": 3244, "field": 3245, "\u0120terror": 3246, "raz": 3247, "\u0120Bo": 3248, "\u0120meeting": 3249, "\u0120Park": 3250, "\u0120arrest": 3251, "\u0120fear": 3252, "\u0120aw": 3253, "\u0120Val": 3254, "oring": 3255, "',": 3256, "\u0120extreme": 3257, "arr": 3258, "\u0120workers": 3259, "After": 3260, "\u012031": 3261, "net": 3262, "ament": 3263, "\u0120directly": 3264, "\u0120population": 3265, "ube": 3266, "\u0120October": 3267, "\u0120IN": 3268, "\u0120January": 3269, "59": 3270, "\u0120David": 3271, "\u0120cross": 3272, "cember": 3273, "\u0120First": 3274, "\u0120message": 3275, "irit": 3276, "\u0120nation": 3277, "\u0120poll": 3278, "isions": 3279, "\u0120answer": 3280, "ny": 3281, "isode": 3282, "\u0120carry": 3283, "\u0120Russia": 3284, "\u0120hear": 3285, "ength": 3286, "roy": 3287, "\u0120natural": 3288, "inally": 3289, "\u0120dog": 3290, "mitted": 3291, "\u0120trade": 3292, "\u0120subst": 3293, "\u0120multiple": 3294, "\u0120Afric": 3295, "\u0120fans": 3296, "\u0120sort": 3297, "\u0120global": 3298, "ication": 3299, "\u0120Wed": 3300, "ara": 3301, "\u0120achie": 3302, "\u0120language": 3303, "vey": 3304, "\u0120tal": 3305, "\u0120necessary": 3306, "\u0120details": 3307, "\u0120sen": 3308, "\u0120Sund": 3309, "\u0120Reg": 3310, "\u0120Rec": 3311, "06": 3312, "\u0120sil": 3313, "ressive": 3314, "\u0120medical": 3315, "unch": 3316, "ornia": 3317, "\u0120und": 3318, "fort": 3319, "ocks": 3320, "\u0120Monday": 3321, "uesday": 3322, "craft": 3323, "77": 3324, "urt": 3325, "\u0120ver": 3326, "\u0120Hill": 3327, "\u0120receive": 3328, "\u0120morning": 3329, "estern": 3330, "\u0120bank": 3331, "\u0120sat": 3332, "irth": 3333, "\u0120High": 3334, "\u0120device": 3335, "\u0120THE": 3336, "\u0120Center": 3337, "\u0120safe": 3338, "\u0120ple": 3339, "\u0120Canada": 3340, "\u0120systems": 3341, "\u0120assist": 3342, "\u0120surv": 3343, "\u0120battle": 3344, "\u0120Soc": 3345, "vertis": 3346, "She": 3347, "\u0120paper": 3348, "\u0120growth": 3349, "\u0120cast": 3350, "Sc": 3351, "\u0120plans": 3352, "lled": 3353, "\u0120parts": 3354, "\u0120wall": 3355, "\u0120movement": 3356, "\u0120practice": 3357, "imately": 3358, "\u0120display": 3359, "\u0120sometimes": 3360, "omp": 3361, "\u0120Paul": 3362, "\u0120Yes": 3363, "king": 3364, "58": 3365, "oly": 3366, "\u0120son": 3367, "\u0120avoid": 3368, "okes": 3369, "\u0120Jew": 3370, "\u0120towards": 3371, "asc": 3372, "\u0120//": 3373, "\u0120Kore": 3374, "\u0120talking": 3375, "\u0120correct": 3376, "\u0120spent": 3377, "icks": 3378, "iable": 3379, "eared": 3380, "\u0120term": 3381, "\u0120wants": 3382, "oming": 3383, "\u0120ut": 3384, "\u0120doub": 3385, "\u0120forces": 3386, "\u0120please": 3387, "69": 3388, "\u0120November": 3389, "atform": 3390, "ondon": 3391, "\u0120ones": 3392, "\u0120immediately": 3393, "\u0120Russian": 3394, "\u0120Met": 3395, "\u0120deg": 3396, "\u0120parents": 3397, "CH": 3398, "\u0120Americans": 3399, "aly": 3400, "\u0120Mod": 3401, "\u0120shown": 3402, "\u0120conditions": 3403, "\u0120stuff": 3404, "\u0120reb": 3405, "\u0120Your": 3406, "\u0120includes": 3407, "nown": 3408, "\u0120Sam": 3409, "\u0120experien": 3410, "mission": 3411, "\u0120Even": 3412, "aught": 3413, "\u0120announced": 3414, "\u0120Republican": 3415, "\u0120determin": 3416, "\u0120described": 3417, "\u0120County": 3418, "()": 3419, "\u0120door": 3420, "\u0120changed": 3421, "\u0120neigh": 3422, "\u0120Here": 3423, "\u0120clean": 3424, "\u0120pan": 3425, "\u0120December": 3426, "\u0120European": 3427, "iring": 3428, "apter": 3429, "\u0120club": 3430, "\u0120Tuesday": 3431, "\u0120paid": 3432, "\u0120Net": 3433, "\u0120attacks": 3434, "\u0120characters": 3435, "\u0120alone": 3436, "\u0120director": 3437, "dom": 3438, "\u012035": 3439, "\u0120load": 3440, "\u0120rout": 3441, "\u0120California": 3442, "\u0120finally": 3443, "\u0120rac": 3444, "\u0120contr": 3445, "\u0120exactly": 3446, "resh": 3447, "pri": 3448, "\u0120Islam": 3449, "\u0120nature": 3450, "\u0120career": 3451, "\u0120latest": 3452, "\u0120convers": 3453, "\u0120Sl": 3454, "pose": 3455, "cient": 3456, "\u0120Inc": 3457, "ivity": 3458, "88": 3459, "\u0120Att": 3460, "\u0120Mor": 3461, "nesday": 3462, "\u0120weight": 3463, "ken": 3464, "\u0120note": 3465, "\u0120teams": 3466, "\u0120\\": 3467, "airs": 3468, "\u0120Green": 3469, "\u0120hundred": 3470, "onent": 3471, "\u0120streng": 3472, "\u0120consist": 3473, "icated": 3474, "\u0120regul": 3475, "\u0120lic": 3476, "astic": 3477, "\u0120ten": 3478, "ursday": 3479, "elligence": 3480, "ously": 3481, "\u0120UK": 3482, "BI": 3483, "\u0120costs": 3484, "\u0120independ": 3485, "\u0120AP": 3486, "\u0120normal": 3487, "\u0120hom": 3488, "\u0120obvious": 3489, "\u0120swe": 3490, "\u0120star": 3491, "\u0120ready": 3492, "acher": 3493, "\u0120implement": 3494, "gest": 3495, "\u0120song": 3496, "\u0120Get": 3497, "\u0120Lab": 3498, "\u0120interesting": 3499, "using": 3500, "\u0120giving": 3501, "\u0120Sunday": 3502, "\u0120etc": 3503, "\u0120middle": 3504, "\u0120remember": 3505, "right": 3506, "osition": 3507, "utions": 3508, "\u0120max": 3509, "46": 3510, "\u0120yourself": 3511, "\u0120demand": 3512, "\u0120treatment": 3513, "\u0120danger": 3514, "\u0120Cons": 3515, "\u0120guy": 3516, "\u0120British": 3517, "\u0120physical": 3518, "\u0120related": 3519, "\u0120remain": 3520, "\u0120couldn": 3521, "\u0120refer": 3522, "\u0120citiz": 3523, "box": 3524, "ENT": 3525, "board": 3526, "\u0120inn": 3527, "IG": 3528, "ero": 3529, "\u0120Street": 3530, "ospital": 3531, "rench": 3532, "chers": 3533, "\u0120stra": 3534, "OL": 3535, "ager": 3536, "\u0120AN": 3537, "\u0120easily": 3538, "IA": 3539, "enge": 3540, "iny": 3541, "\u0120clos": 3542, "ocked": 3543, "\u0120uses": 3544, "\u0120Coun": 3545, "Im": 3546, "uild": 3547, "??": 3548, "more": 3549, "\u0120ang": 3550, "\u0120write": 3551, "olute": 3552, "57": 3553, "\u0120leader": 3554, "\u0120reading": 3555, "": 3784, "\u0120figure": 3785, "\u0120disapp": 3786, "enty": 3787, "\u0120software": 3788, "\u0120ult": 3789, "\u0120officers": 3790, "New": 3791, "Is": 3792, "\u0120remains": 3793, "\u0120India": 3794, "\u0120psych": 3795, "rief": 3796, "\u0120cat": 3797, "esc": 3798, "\u0120observ": 3799, "\u0120stage": 3800, "\u0120Dark": 3801, "\u0120enter": 3802, "change": 3803, "\u0120passed": 3804, "\u0120despite": 3805, "\u0120Out": 3806, "\u0120movie": 3807, "rs": 3808, "\u0120voice": 3809, "mine": 3810, "\u0120Play": 3811, "\u0120toward": 3812, "\u0120Ter": 3813, "\u0120region": 3814, "\u0120values": 3815, "orters": 3816, "\u0120mount": 3817, "\u0120officer": 3818, "\u0120Other": 3819, "ban": 3820, "\u0120hous": 3821, "wood": 3822, "room": 3823, "IV": 3824, "\u0120Sun": 3825, "see": 3826, "\u0120Over": 3827, "rog": 3828, "90": 3829, "\u0120lay": 3830, "\u0120Tur": 3831, "awn": 3832, "\u0120pressure": 3833, "\u0120Sub": 3834, "\u0120books": 3835, "edom": 3836, "\u0120Sand": 3837, "AA": 3838, "ago": 3839, "\u0120reasons": 3840, "ford": 3841, "\u0120activity": 3842, "UT": 3843, "Now": 3844, "\u0120Senate": 3845, "cell": 3846, "night": 3847, "\u0120calls": 3848, "inter": 3849, "\u0120letter": 3850, "\u0120Rob": 3851, "\u0120Je": 3852, "\u0120choose": 3853, "\u0120Law": 3854, "Get": 3855, "Be": 3856, "\u0120rob": 3857, "\u0120types": 3858, "\u0120platform": 3859, "\u0120quarter": 3860, "RA": 3861, "\u0120Time": 3862, "\u0120maybe": 3863, "\u0120Cr": 3864, "95": 3865, "pre": 3866, "\u0120moving": 3867, "\u0120lif": 3868, "\u0120gold": 3869, "\u0120som": 3870, "\u0120patients": 3871, "\u0120truth": 3872, "\u0120Ke": 3873, "urance": 3874, "antly": 3875, "mar": 3876, "\u0120charge": 3877, "\u0120Great": 3878, "\u0120cele": 3879, "--------------------------------": 3880, "\u0120rock": 3881, "roid": 3882, "ancy": 3883, "\u0120credit": 3884, "aud": 3885, "By": 3886, "\u0120Every": 3887, "\u0120moved": 3888, "inger": 3889, "ribution": 3890, "\u0120names": 3891, "\u0120straight": 3892, "\u0120Health": 3893, "\u0120Well": 3894, "\u0120feature": 3895, "\u0120rule": 3896, "\u0120sche": 3897, "inated": 3898, "\u0120Michael": 3899, "berg": 3900, "41": 3901, "iled": 3902, "band": 3903, "\u0120click": 3904, "\u0120Angel": 3905, "onents": 3906, "\u00c2\u0143": 3907, "\u0120Iraq": 3908, "\u0120Saturday": 3909, "\u0120aware": 3910, "part": 3911, "\u0120pattern": 3912, "OW": 3913, "\u0120Let": 3914, "\u0120grad": 3915, "igned": 3916, "\u0120associated": 3917, "\u0120style": 3918, "no": 3919, "iation": 3920, "aith": 3921, "ilies": 3922, "\u0120stories": 3923, "uration": 3924, "\u0120individuals": 3925, "\u0120\u00e2\u0122\u00a6": 3926, "miss": 3927, "\u0120Associ": 3928, "ishing": 3929, "aby": 3930, "\u0120summer": 3931, "\u0120Ben": 3932, "\u012032": 3933, "\u0120arch": 3934, "uty": 3935, "\u0120Texas": 3936, "hol": 3937, "\u0120fully": 3938, "\u0120mill": 3939, "\u0120followed": 3940, "\u0120Bill": 3941, "\u0120Indian": 3942, "\u0120Secret": 3943, "\u0120Bel": 3944, "\u0120February": 3945, "\u0120jobs": 3946, "\u0120seemed": 3947, "\u0120Govern": 3948, "ipped": 3949, "\u0120reality": 3950, "\u0120lines": 3951, "\u0120park": 3952, "\u0120measure": 3953, "\u0120Our": 3954, "IM": 3955, "\u0120brother": 3956, "\u0120growing": 3957, "\u0120ban": 3958, "\u0120estim": 3959, "\u0120cry": 3960, "\u0120School": 3961, "\u0120mechan": 3962, "\u0120OF": 3963, "\u0120Windows": 3964, "\u0120rates": 3965, "\u0120Oh": 3966, "\u0120positive": 3967, "\u0120culture": 3968, "istics": 3969, "ica": 3970, "\u0120har": 3971, "ya": 3972, "itely": 3973, "ipp": 3974, "\u0120map": 3975, "encies": 3976, "\u0120William": 3977, "II": 3978, "akers": 3979, "56": 3980, "\u0120Mart": 3981, "\u0120Rem": 3982, "\u0120altern": 3983, "itude": 3984, "\u0120coach": 3985, "rowd": 3986, "Don": 3987, "\u0120kids": 3988, "\u0120journal": 3989, "\u0120corpor": 3990, "\u0120false": 3991, "\u0120web": 3992, "\u0120sleep": 3993, "\u0120contain": 3994, "\u0120sto": 3995, "\u0120bed": 3996, "iverse": 3997, "\u0120Rich": 3998, "\u0120Chinese": 3999, "\u0120pun": 4000, "\u0120meant": 4001, "known": 4002, "\u0120notice": 4003, "\u0120favorite": 4004, "aven": 4005, "\u0120condition": 4006, "\u0120purpose": 4007, "))": 4008, "\u0120organization": 4009, "\u0120challeng": 4010, "\u0120manufact": 4011, "\u0120susp": 4012, "\u0120Ac": 4013, "\u0120critic": 4014, "unes": 4015, "uclear": 4016, "\u0120mer": 4017, "vention": 4018, "\u012080": 4019, "\u0120mist": 4020, "\u0120Us": 4021, "\u0120Tor": 4022, "http": 4023, "olf": 4024, "\u0120larger": 4025, "\u0120advant": 4026, "\u0120resear": 4027, "\u0120actions": 4028, "ml": 4029, "\u0120kept": 4030, "\u0120aim": 4031, ",'": 4032, "col": 4033, "\u0120benefits": 4034, "ifying": 4035, "\u0120actual": 4036, "\u0120International": 4037, "\u0120vehicle": 4038, "\u0120chief": 4039, "\u0120efforts": 4040, "\u0120League": 4041, "\u0120Most": 4042, "\u0120wait": 4043, "\u0120adult": 4044, "\u0120overall": 4045, "\u0120speech": 4046, "\u0120highly": 4047, "\u0120female": 4048, "\u0120error": 4049, "\u0120effective": 4050, "54": 4051, "\u0120encour": 4052, "well": 4053, "\u0120failed": 4054, "\u0120conserv": 4055, "\u0120programs": 4056, "\u0120trou": 4057, "\u0120ahead": 4058, "500": 4059, "vertisement": 4060, "IP": 4061, "\u0120Found": 4062, "pir": 4063, "\u0120%": 4064, "\u0120crime": 4065, "ander": 4066, "\u0120location": 4067, "\u0120Iran": 4068, "\u0120behavior": 4069, "azing": 4070, "\u0120rare": 4071, "\u0120emb": 4072, "\u0120caused": 4073, "\u0120ship": 4074, "\u0120active": 4075, "\u0120contribut": 4076, "\u0120green": 4077, "\u0120acqu": 4078, "\u0120reflect": 4079, "venue": 4080, "\u0120firm": 4081, "\u0120birth": 4082, "].": 4083, "\u0120clearly": 4084, "\u0120emot": 4085, "\u0120agency": 4086, "riage": 4087, "\u0120memory": 4088, "98": 4089, "SA": 4090, "\u0120See": 4091, "acing": 4092, "CC": 4093, "\u0120biggest": 4094, "\u0120rap": 4095, "\u0120basic": 4096, "\u0120band": 4097, "eat": 4098, "\u0120suspect": 4099, "\u0120Mac": 4100, "\u012090": 4101, "mark": 4102, "istan": 4103, "\u0120spread": 4104, "ams": 4105, "ki": 4106, "asy": 4107, "rav": 4108, "\u0120Rober": 4109, "\u0120demonstr": 4110, "rated": 4111, "\u0120absolute": 4112, "\u0120places": 4113, "\u0120impl": 4114, "ibrary": 4115, "\u0120cards": 4116, "\u0120destroy": 4117, "\u0120virt": 4118, "vere": 4119, "\u0120appeared": 4120, "yan": 4121, "point": 4122, "\u0120beg": 4123, "\u0120temper": 4124, "spe": 4125, "anted": 4126, "ears": 4127, "\u0120Direct": 4128, "\u0120length": 4129, "\u0120blog": 4130, "amb": 4131, "\u0120integ": 4132, "\u0120resources": 4133, "acc": 4134, "iful": 4135, "\u0120spot": 4136, "\u0120forced": 4137, "\u0120thousands": 4138, "\u0120Minister": 4139, "\u0120qual": 4140, "\u0120French": 4141, "atically": 4142, "\u0120generally": 4143, "\u0120drink": 4144, "\u0120thus": 4145, "IL": 4146, "odes": 4147, "\u0120appropri": 4148, "\u0120Read": 4149, "\u0120whom": 4150, "\u0120eye": 4151, "\u0120college": 4152, "\u012045": 4153, "irection": 4154, "\u0120ensure": 4155, "\u0120apparent": 4156, "iders": 4157, "\u0120religious": 4158, "\u0120minor": 4159, "olic": 4160, "\u0120tro": 4161, "\u0120Why": 4162, "ribute": 4163, "met": 4164, "\u0120primary": 4165, "\u0120developed": 4166, "\u0120peace": 4167, "\u0120skin": 4168, "ste": 4169, "ava": 4170, "\u0120blue": 4171, "\u0120families": 4172, "\u0120ir": 4173, "\u0120apply": 4174, "\u0120inform": 4175, "\u0120Smith": 4176, "CT": 4177, "ii": 4178, "\u0120limit": 4179, "\u0120resist": 4180, "................": 4181, "umn": 4182, "\u0120conflic": 4183, "\u0120twe": 4184, "udd": 4185, "\u0120Tom": 4186, "\u0120liter": 4187, "que": 4188, "bon": 4189, "\u0120hair": 4190, "\u0120eventually": 4191, "\u0120pus": 4192, "\u0120helped": 4193, "\u0120agg": 4194, "orney": 4195, "\u0120Apple": 4196, "\u0120fit": 4197, "\u0120Sur": 4198, "\u0120prem": 4199, "\u0120sales": 4200, "\u0120seconds": 4201, "\u0120strength": 4202, "\u0120feeling": 4203, "\u00bf\u00bd": 4204, "\u0120tour": 4205, "\u0120knows": 4206, "oom": 4207, "\u0120exerc": 4208, "\u0120somew": 4209, "\u00ef\u00bf\u00bd": 4210, ">>": 4211, "\u0120spokes": 4212, "\u0120ideas": 4213, "\u0120regist": 4214, "soft": 4215, "\u0120Del": 4216, "\u0120PC": 4217, "\u0120propos": 4218, "\u0120launch": 4219, "\u0120bottom": 4220, "TH": 4221, "\u0120Please": 4222, "vest": 4223, "itz": 4224, "\u0120Inter": 4225, "\u0120script": 4226, "\u0120rat": 4227, "arning": 4228, "\u0120il": 4229, "\u0120Jer": 4230, "\u0120Are": 4231, "\u0120whatever": 4232, "oken": 4233, "cience": 4234, "\u0120mode": 4235, "\u0120agree": 4236, "\u0120sources": 4237, "\u0120initial": 4238, "\u0120restrict": 4239, "\u0120wonder": 4240, "usion": 4241, "####": 4242, "\u0120Sil": 4243, "ville": 4244, "\u0120burn": 4245, "tw": 4246, "asion": 4247, "\u0120\u00c2\u00a3": 4248, "\u0120nor": 4249, "uing": 4250, "\u0120reached": 4251, "\u0120sun": 4252, "\u0120categ": 4253, "igration": 4254, "\u0120cook": 4255, "\u0120promot": 4256, "\u0120male": 4257, "\u0120climate": 4258, "\u0120fix": 4259, "\u0120alleged": 4260, "UR": 4261, "alled": 4262, "\u0120images": 4263, "Cont": 4264, "ota": 4265, "\u0120schools": 4266, "ios": 4267, "\u0120drop": 4268, "\u0120stream": 4269, "\u0120Mo": 4270, "\u0120previously": 4271, "aling": 4272, "\u0120pet": 4273, "\u0120double": 4274, "\u0120(@": 4275, "annel": 4276, "\u0120default": 4277, "ties": 4278, "\u0120rank": 4279, "\u0120Dec": 4280, "\u0120Council": 4281, "\u0120weapon": 4282, "\u0120stock": 4283, "\u0120analy": 4284, "\u0120Str": 4285, "\u0120picture": 4286, "\u0120Police": 4287, "ference": 4288, "\u0120century": 4289, "\u0120citizens": 4290, "\u0120onto": 4291, "\u0120expand": 4292, "\u0120hero": 4293, "\u0120Sol": 4294, "\u0120wild": 4295, "\u0120update": 4296, "\u0120customers": 4297, "ront": 4298, "def": 4299, "\u0120lik": 4300, "\u0120criminal": 4301, "\u0120Christian": 4302, "SP": 4303, "76": 4304, "\u0120leaving": 4305, "\u0120otherwise": 4306, "\u0120Dist": 4307, "\u0120basis": 4308, "52": 4309, "53": 4310, "icip": 4311, "\u0120Ber": 4312, "\u0120recommend": 4313, "\u0120floor": 4314, "\u0120crowd": 4315, "oles": 4316, "\u012070": 4317, "\u0120central": 4318, "\u0120Ev": 4319, "\u0120dream": 4320, "\u0120download": 4321, "\u0120confir": 4322, "\u0120Thom": 4323, "\u0120window": 4324, "\u0120happens": 4325, "\u0120unit": 4326, "\u0120tend": 4327, "\u0120spl": 4328, "\u0120becomes": 4329, "\u0120fighting": 4330, "\u0120predict": 4331, "\u0120Press": 4332, "\u0120Power": 4333, "\u0120heavy": 4334, "aked": 4335, "\u0120fan": 4336, "orter": 4337, "ategy": 4338, "BA": 4339, "izes": 4340, "\u0120spend": 4341, "Here": 4342, "\u01202007": 4343, "\u0120adop": 4344, "\u0120Ham": 4345, "\u0120football": 4346, "\u0120Port": 4347, "oday": 4348, "51": 4349, "ampions": 4350, "\u0120transfer": 4351, "ht": 4352, "\u012038": 4353, "term": 4354, "acity": 4355, "\u0120bur": 4356, "],": 4357, "ternal": 4358, "rig": 4359, "but": 4360, "\u0120therefore": 4361, "\u0120Because": 4362, "resp": 4363, "rey": 4364, "\u0120mission": 4365, "Some": 4366, "\u0120noted": 4367, "\u0120assum": 4368, "\u0120disease": 4369, "\u0120edit": 4370, "\u0120progress": 4371, "rd": 4372, "\u0120Brown": 4373, "ocal": 4374, "\u0120adding": 4375, "\u0120raised": 4376, "\u0120Any": 4377, "\u0120tick": 4378, "\u0120seeing": 4379, "\u0120People": 4380, "\u0120agreement": 4381, "\u0120server": 4382, "\u0120wat": 4383, "\u0120debate": 4384, "\u0120supposed": 4385, "iling": 4386, "\u0120largest": 4387, "\u0120successful": 4388, "\u0120Pri": 4389, "\u0120Democratic": 4390, "\u0120jump": 4391, "\u0120Syria": 4392, "\u0120owners": 4393, "\u0120offers": 4394, "\u0120shooting": 4395, "\u0120effic": 4396, "sey": 4397, "\u0120haven": 4398, "verse": 4399, "tered": 4400, "\u0120Light": 4401, "imal": 4402, "\u0120Big": 4403, "\u0120defend": 4404, "\u0120beat": 4405, "\u0120records": 4406, "%)": 4407, "\u0120scen": 4408, "\u0120employees": 4409, "\u0120devices": 4410, "hem": 4411, "\u0120commer": 4412, "\u0120Mex": 4413, "\u0120benefit": 4414, "\u0120Prof": 4415, "\u0120illeg": 4416, "\u0120surface": 4417, "\u0120Also": 4418, "\u0120harm": 4419, "ingly": 4420, "wide": 4421, "\u0120Alex": 4422, "\u0120shut": 4423, "\u0120Cur": 4424, "\u0120lose": 4425, "pm": 4426, "\u0120challenge": 4427, "semb": 4428, "\u0120station": 4429, "\u0120intelligence": 4430, "\u0120accur": 4431, "\u0120Flor": 4432, "\u0120requires": 4433, "\u0120Mal": 4434, "bum": 4435, "\u0120hospital": 4436, "\u0120spirit": 4437, "\u0120offered": 4438, "\u0120produce": 4439, "\u0120Commun": 4440, "\u0120creating": 4441, "\u0120cris": 4442, "spect": 4443, "\u0120ended": 4444, "\u0120daily": 4445, "\u0120voters": 4446, "lands": 4447, "ias": 4448, "ih": 4449, "ona": 4450, "\u0120smart": 4451, "\u0120Office": 4452, "\u0120Lord": 4453, "rial": 4454, "\u0120Internet": 4455, "\u0120circum": 4456, "\u0120extremely": 4457, "'.": 4458, "\u0120opinion": 4459, "\u0120Mil": 4460, "\u0120gain": 4461, "BS": 4462, "\u0120Fin": 4463, "yp": 4464, "\u0120useful": 4465, "\u0120budget": 4466, "\u0120comfort": 4467, "isf": 4468, "\u0120background": 4469, "eline": 4470, "\u0120episode": 4471, "\u0120enemy": 4472, "\u0120trial": 4473, "\u0120establish": 4474, "date": 4475, "\u0120Cap": 4476, "\u0120continues": 4477, "\u0120showing": 4478, "\u0120Union": 4479, "with": 4480, "\u0120posted": 4481, "\u0120System": 4482, "\u0120eat": 4483, "rian": 4484, "\u0120rise": 4485, "\u0120Germany": 4486, "ils": 4487, "\u0120signed": 4488, "\u0120vill": 4489, "\u0120grand": 4490, "mor": 4491, "\u0120England": 4492, "\u0120projects": 4493, "umber": 4494, "\u0120conference": 4495, "za": 4496, "\u0120responsible": 4497, "\u0120Arab": 4498, "\u0120learned": 4499, "\u00e2\u0122\u0136\u00e2\u0122\u0136": 4500, "ipping": 4501, "\u0120George": 4502, "OC": 4503, "\u0120returned": 4504, "\u0120Australia": 4505, "\u0120brief": 4506, "Qu": 4507, "\u0120brand": 4508, "illing": 4509, "abled": 4510, "\u0120highest": 4511, "\u0120train": 4512, "\u0120Commission": 4513, "while": 4514, "\u0120nom": 4515, "ception": 4516, "\u0120mut": 4517, "\u0120Blue": 4518, "\u0120incident": 4519, "vant": 4520, "86": 4521, "\u0120ID": 4522, "\u0120nuclear": 4523, "74": 4524, "\u0120Like": 4525, "\u0120RE": 4526, "\u0120Micro": 4527, "li": 4528, "mail": 4529, "\u0120charges": 4530, "89": 4531, "\u0120adjust": 4532, "ado": 4533, "\u0120earth": 4534, "NA": 4535, "\u0120prices": 4536, "PA": 4537, "\u0120draft": 4538, "\u0120runs": 4539, "\u0120candidate": 4540, "enses": 4541, "\u0120management": 4542, "\u0120Phil": 4543, "\u0120Miss": 4544, "\u0120teach": 4545, "gram": 4546, "\u0120understanding": 4547, "ait": 4548, "icago": 4549, "Add": 4550, "\u0120Ep": 4551, "secut": 4552, "\u0120separate": 4553, "\u0120instance": 4554, "\u0120eth": 4555, "\u0120unless": 4556, "********": 4557, "\u0120Fore": 4558, "inate": 4559, "\u0120operations": 4560, "Sp": 4561, "\u0120faith": 4562, "gar": 4563, "\u0120Church": 4564, "ronic": 4565, "\u0120config": 4566, "osure": 4567, "\u0120activities": 4568, "\u0120traditional": 4569, "\u012036": 4570, "\u0120direction": 4571, "\u0120machine": 4572, "\u0120surround": 4573, "\u0120push": 4574, "unction": 4575, "\u0120EU": 4576, "\u0120easier": 4577, "\u0120argument": 4578, "GB": 4579, "\u0120micro": 4580, "\u0120spending": 4581, "izations": 4582, "\u0120theory": 4583, "adow": 4584, "\u0120calling": 4585, "\u0120Last": 4586, "\u0120der": 4587, "\u0120influence": 4588, "\u0120commit": 4589, "\u0120photo": 4590, "\u0120unc": 4591, "istry": 4592, "gn": 4593, "aste": 4594, "acks": 4595, "\u0120disp": 4596, "ady": 4597, "do": 4598, "\u0120Good": 4599, "\u0120`": 4600, "\u0120wish": 4601, "\u0120revealed": 4602, "\u00c2\u0142\u00c2\u0142": 4603, "lig": 4604, "\u0120enforce": 4605, "\u0120Committee": 4606, "\u0120chem": 4607, "\u0120miles": 4608, "\u0120interested": 4609, "\u0120solution": 4610, "icy": 4611, "inct": 4612, "\u0120->": 4613, "\u0120Det": 4614, "\u0120removed": 4615, "\u0120compar": 4616, "eah": 4617, "\u0120plant": 4618, "\u0120Since": 4619, "\u0120achieve": 4620, "\u0120advantage": 4621, "\u0120slightly": 4622, "bing": 4623, "\u0120placed": 4624, "under": 4625, "2015": 4626, "\u0120Mad": 4627, "\u0120tim": 4628, "oses": 4629, "\u0120cru": 4630, "\u0120Rock": 4631, "\u0120mostly": 4632, "\u0120negative": 4633, "\u0120setting": 4634, "\u0120produced": 4635, "\u0120mur": 4636, "\u0120connection": 4637, "\u0120Mer": 4638, "\u0120driver": 4639, "\u0120executive": 4640, "\u0120assault": 4641, "\u0120born": 4642, "\u0120Ver": 4643, "tained": 4644, "\u0120structure": 4645, "\u0120reduce": 4646, "\u0120decades": 4647, "\u0120ded": 4648, "uke": 4649, "\u0120Many": 4650, "idden": 4651, "\u0120league": 4652, "Se": 4653, "\u0120join": 4654, "\u0120disco": 4655, "\u0120die": 4656, "cks": 4657, "actions": 4658, "\u0120assess": 4659, "agn": 4660, "\u0120goals": 4661, "ours": 4662, "IR": 4663, "\u0120senior": 4664, "iller": 4665, "mod": 4666, "ipment": 4667, "ocol": 4668, "uy": 4669, "\u0120Que": 4670, "\u0120parties": 4671, "irgin": 4672, "\u0120learning": 4673, "itable": 4674, "\u0120street": 4675, "\u0120camera": 4676, "App": 4677, "\u0120skills": 4678, "bre": 4679, "cious": 4680, "\u0120celebr": 4681, "\u0120Franc": 4682, "\u0120existing": 4683, "\u0120willing": 4684, "lor": 4685, "\u0120id": 4686, "\u0120Space": 4687, "\u0120critical": 4688, "\u0120La": 4689, "ortunately": 4690, "\u0120serve": 4691, "\u0120cold": 4692, "\u0120species": 4693, "TS": 4694, "\u0120animals": 4695, "\u0120Bay": 4696, "\u0120older": 4697, "\u0120Under": 4698, "estic": 4699, "\u0120Tre": 4700, "\u0120teacher": 4701, "\u0120prefer": 4702, "vis": 4703, "\u0120thread": 4704, "\u0120Matt": 4705, "\u0120manager": 4706, "\u00e3\u0125\u00bb": 4707, "\u0120professional": 4708, "\u0120Vol": 4709, "\u0120notes": 4710, "These": 4711, "ula": 4712, "\u0120fresh": 4713, "ented": 4714, "uzz": 4715, "edy": 4716, "clusion": 4717, "\u0120Rel": 4718, "\u0120doubt": 4719, "EO": 4720, "\u0120opened": 4721, "\u0120Bit": 4722, "Advertisement": 4723, "\u0120guess": 4724, "\u0120UN": 4725, "\u0120sequ": 4726, "\u0120explain": 4727, "otten": 4728, "\u0120attract": 4729, "aks": 4730, "\u0120string": 4731, "\u0120context": 4732, "ossible": 4733, "\u0120Republicans": 4734, "\u0120solid": 4735, "\u0120cities": 4736, "\u0120asking": 4737, "\u0120random": 4738, "ups": 4739, "uries": 4740, "arant": 4741, "dden": 4742, "gl": 4743, "\u0120Florida": 4744, "\u0120depend": 4745, "\u0120Scott": 4746, "\u012033": 4747, "\u0120iT": 4748, "icon": 4749, "\u0120mentioned": 4750, "\u01202000": 4751, "\u0120claimed": 4752, "\u0120definitely": 4753, "ulf": 4754, "\u0120core": 4755, "\u0120opening": 4756, "\u0120Const": 4757, "which": 4758, "\u0120Tra": 4759, "AG": 4760, "72": 4761, "\u0120believed": 4762, "ada": 4763, "\u012048": 4764, "\u0120Security": 4765, "yright": 4766, "\u0120Pet": 4767, "\u0120Lou": 4768, "\u0120holding": 4769, "================": 4770, "\u0120ice": 4771, "\u0120brow": 4772, "\u0120authorities": 4773, "host": 4774, "word": 4775, "\u0120score": 4776, "\u0120Div": 4777, "\u0120cells": 4778, "\u0120transl": 4779, "\u0120neighbor": 4780, "\u0120remove": 4781, "uct": 4782, "\u0120district": 4783, "\u0120According": 4784, "\u0120worse": 4785, "\u0120concerns": 4786, "\u0120presidential": 4787, "\u0120policies": 4788, "\u0120Hall": 4789, "73": 4790, "\u0120hus": 4791, "AY": 4792, "\u01202006": 4793, "\u0120Jud": 4794, "\u0120independent": 4795, "\u0120Justice": 4796, "iliar": 4797, "print": 4798, "ighter": 4799, "\u0120protection": 4800, "zen": 4801, "\u0120sudden": 4802, "house": 4803, "\u0120Jes": 4804, "PR": 4805, "\u0120Inf": 4806, "\u0120bul": 4807, "\u0120_": 4808, "\u0120Service": 4809, "\u0120PR": 4810, "\u0120strategy": 4811, "ffect": 4812, "\u0120girls": 4813, "\u0120missing": 4814, "oyal": 4815, "\u0120Team": 4816, "ulated": 4817, "\u0120dat": 4818, "\u0120politics": 4819, "abor": 4820, "According": 4821, "\u0120spell": 4822, "\u0120graph": 4823, "orthern": 4824, "TC": 4825, "Ab": 4826, "\u0120labor": 4827, "isher": 4828, "\u0120kick": 4829, "\u0120iTunes": 4830, "\u0120steps": 4831, "poses": 4832, "\u0120smaller": 4833, "En": 4834, "bert": 4835, "\u0120roll": 4836, "\u0120researchers": 4837, "\u0120closed": 4838, "\u0120transport": 4839, "\u0120lawy": 4840, "________________": 4841, "\u0120Chicago": 4842, "\u0120aspect": 4843, "\u0120none": 4844, "\u0120marriage": 4845, "96": 4846, "\u0120elements": 4847, "\u0120Fre": 4848, "\u0120Sal": 4849, "\u0120dram": 4850, "FC": 4851, "top": 4852, "equ": 4853, "\u0120hearing": 4854, "\u0120supported": 4855, "\u0120testing": 4856, "cohol": 4857, "\u0120massive": 4858, "\u0120stick": 4859, "\u0120guard": 4860, "isco": 4861, "phone": 4862, "From": 4863, "However": 4864, "\u0120border": 4865, "\u0120copy": 4866, "ography": 4867, "list": 4868, "71": 4869, "\u0120owner": 4870, "class": 4871, "ruit": 4872, "rate": 4873, "\u0120Once": 4874, "\u0120digital": 4875, "\u0120task": 4876, "ERS": 4877, "\u0120incred": 4878, "tes": 4879, "++": 4880, "\u0120France": 4881, "\u0120breat": 4882, "owl": 4883, "\u0120issued": 4884, "\u0120Western": 4885, "\u0120detect": 4886, "\u0120partners": 4887, "\u0120shared": 4888, "\u0120Call": 4889, "\u0120cancer": 4890, "ache": 4891, "ribe": 4892, "\u0120explained": 4893, "\u0120heat": 4894, "{\"": 4895, "\u0120investment": 4896, "\u0120Book": 4897, "\u0120wood": 4898, "\u0120tools": 4899, "\u0120Although": 4900, "\u0120belief": 4901, "\u0120crisis": 4902, "\u0120ge": 4903, "\u0120MP": 4904, "\u0120operation": 4905, "type": 4906, "~~": 4907, "ga": 4908, "\u0120contains": 4909, "anta": 4910, "\u0120express": 4911, "\u0120Group": 4912, "\u0120Journal": 4913, "ka": 4914, "\u0120amb": 4915, "\u0120USA": 4916, "\u0120finding": 4917, "\u0120funding": 4918, "how": 4919, "\u0120established": 4920, "ideos": 4921, "\u0120degree": 4922, "\u0120dangerous": 4923, "anging": 4924, "\u0120freedom": 4925, "pport": 4926, "outhern": 4927, "\u0120church": 4928, "\u0120catch": 4929, "\u0120Two": 4930, "\u0120presence": 4931, "\u0120Guard": 4932, "Up": 4933, "\u0120authority": 4934, "\u0120Project": 4935, "\u0120button": 4936, "\u0120consequ": 4937, "\u0120valid": 4938, "\u0120weak": 4939, "\u0120starts": 4940, "\u0120reference": 4941, "\u0120Mem": 4942, "\")": 4943, "UN": 4944, "orage": 4945, "\u0120Open": 4946, "\u0120collection": 4947, "ym": 4948, "gency": 4949, "\u0120beautiful": 4950, "ros": 4951, "\u0120tells": 4952, "\u0120waiting": 4953, "nel": 4954, "\u0120providing": 4955, "\u0120Democrats": 4956, "\u0120daughter": 4957, "\u0120master": 4958, "\u0120purposes": 4959, "\u0120Japanese": 4960, "\u0120equal": 4961, "\u0120turns": 4962, "\u0120documents": 4963, "\u0120watching": 4964, "Res": 4965, "\u0120ran": 4966, "2014": 4967, "\u0120reject": 4968, "\u0120Korea": 4969, "\u0120victims": 4970, "Level": 4971, "erences": 4972, "\u0120witness": 4973, "\u012034": 4974, "\u0120reform": 4975, "coming": 4976, "\u0120occup": 4977, "\u0120caught": 4978, "\u0120traffic": 4979, "ading": 4980, "\u0120models": 4981, "ario": 4982, "\u0120served": 4983, "\u0120batter": 4984, "uate": 4985, "\u0120Secretary": 4986, "\u0120agreed": 4987, "\u0120truly": 4988, "ynam": 4989, "\u0120Ret": 4990, "\u0120units": 4991, "\u0120Research": 4992, "hand": 4993, "azine": 4994, "\u0120Mike": 4995, "\u0120variety": 4996, "otal": 4997, "\u0120amazing": 4998, "\u0120confirmed": 4999, "\u0120entirely": 5000, "\u0120purchase": 5001, "\u0120element": 5002, "\u0120cash": 5003, "\u0120determine": 5004, "De": 5005, "\u0120cars": 5006, "\u0120Wall": 5007, "\u00e2\u0138": 5008, "\u0120views": 5009, "\u0120drugs": 5010, "\u0120department": 5011, "\u0120Step": 5012, "uit": 5013, "\u012039": 5014, "asure": 5015, "\u0120Class": 5016, "\u0120covered": 5017, "\u0120Bank": 5018, "\u0120mere": 5019, "uana": 5020, "\u0120multi": 5021, "\u0120mix": 5022, "\u0120unlike": 5023, "levision": 5024, "\u0120stopped": 5025, "\u0120sem": 5026, "\u0120Gal": 5027, "ules": 5028, "\u0120wel": 5029, "\u0120Johnson": 5030, "la": 5031, "\u0120skill": 5032, "\u0120becoming": 5033, "rie": 5034, "\u0120appropriate": 5035, "fe": 5036, "ellow": 5037, "\u0120Prot": 5038, "ulate": 5039, "ocation": 5040, "\u0120weekend": 5041, "odies": 5042, "\u0120sites": 5043, "\u0120animal": 5044, "\u0120Tim": 5045, "\u0120scale": 5046, "\u0120charged": 5047, "\u0120instruct": 5048, "illa": 5049, "\u0120methods": 5050, "\u0120cert": 5051, "\u0120judge": 5052, "\u0120Hel": 5053, "\u0120dollars": 5054, "\u0120standing": 5055, "\u0120Squ": 5056, "\u0120debt": 5057, "liam": 5058, "\u0120driving": 5059, "\u0120Sum": 5060, "\u0120Edition": 5061, "\u0120album": 5062, "andon": 5063, "IF": 5064, "\u0120Uk": 5065, "63": 5066, "ader": 5067, "\u0120commercial": 5068, "esh": 5069, "\u0120Government": 5070, "\u0120discovered": 5071, "\u0120output": 5072, "\u0120Hillary": 5073, "\u0120Carol": 5074, "\u01202005": 5075, "\u0120abuse": 5076, "ancing": 5077, "\u0120switch": 5078, "\u0120annual": 5079, "Tw": 5080, "\u0120stated": 5081, "agement": 5082, "inner": 5083, "\u0120democr": 5084, "\u0120residents": 5085, "\u0120allowing": 5086, "\u0120factors": 5087, "odd": 5088, "\u0120fuck": 5089, "emies": 5090, "\u0120occurred": 5091, "oti": 5092, "\u0120north": 5093, "\u0120Public": 5094, "\u0120injury": 5095, "\u0120insurance": 5096, "CL": 5097, "olly": 5098, "\u00e3\u0122": 5099, "\u0120repeated": 5100, "\u0120arms": 5101, "anged": 5102, "\u0120construction": 5103, "\u0120fle": 5104, "PU": 5105, "icians": 5106, "\u0120forms": 5107, "\u0120McC": 5108, "antic": 5109, "\u0120mental": 5110, "pire": 5111, "\u0120equipment": 5112, "\u0120fant": 5113, "\u0120discussion": 5114, "\u0120regarding": 5115, "kin": 5116, "arp": 5117, "\u0120chair": 5118, "ogue": 5119, "\u0120proceed": 5120, "\u0120Id": 5121, "Our": 5122, "\u0120murder": 5123, "Man": 5124, "\u012049": 5125, "asp": 5126, "\u0120supply": 5127, "\u0120input": 5128, "\u0120wealth": 5129, "liament": 5130, "\u0120proced": 5131, "orial": 5132, "\u0120Stat": 5133, "\u0120NFL": 5134, "hens": 5135, "\u0120Institute": 5136, "\u0120putting": 5137, "ournament": 5138, "etic": 5139, "\u0120located": 5140, "\u0120kid": 5141, "eria": 5142, "run": 5143, "\u0120princ": 5144, "\u0120!": 5145, "going": 5146, "\u0120Bet": 5147, "\u0120clot": 5148, "\u0120telling": 5149, "\u0120proposed": 5150, "iot": 5151, "orry": 5152, "\u0120funds": 5153, "gment": 5154, "\u0120Life": 5155, "\u0120baby": 5156, "\u0120Back": 5157, "\u0120spoke": 5158, "Image": 5159, "\u0120earn": 5160, "\u0120AT": 5161, "gu": 5162, "\u0120exchange": 5163, "\u0120Lin": 5164, "oving": 5165, "\u0120pair": 5166, "More": 5167, "azon": 5168, "\u0120arrested": 5169, "\u0120killing": 5170, "can": 5171, "\u0120Card": 5172, "yd": 5173, "\u0120identified": 5174, "\u0120mobile": 5175, "\u0120thanks": 5176, "onym": 5177, "\u0120Form": 5178, "\u0120hundreds": 5179, "\u0120Chris": 5180, "\u0120Cat": 5181, "\u0120trend": 5182, "hat": 5183, "\u0120Av": 5184, "oman": 5185, "\u0120electric": 5186, "\u0120Wil": 5187, "SE": 5188, "Of": 5189, "\u0120restaur": 5190, "oted": 5191, "\u0120trig": 5192, "\u0120nine": 5193, "\u0120bomb": 5194, "Why": 5195, "\u00c2\u00af": 5196, "\u0120coverage": 5197, "\u0120appeal": 5198, "\u0120Robert": 5199, "\u0120Sup": 5200, "\u0120finished": 5201, "\u0120flow": 5202, "\u0120deliver": 5203, "\u0120calcul": 5204, "\u0120photos": 5205, "\u0120phil": 5206, "\u0120pieces": 5207, "\u0120appre": 5208, "kes": 5209, "\u0120rough": 5210, "Do": 5211, "\u0120partner": 5212, "\u0120concerned": 5213, "\u012037": 5214, "\u0120Gen": 5215, "Col": 5216, "ctors": 5217, "\u0120=>": 5218, "state": 5219, "\u0120suggested": 5220, "\u0120Force": 5221, "CE": 5222, "\u0120herself": 5223, "\u0120Plan": 5224, "works": 5225, "ooth": 5226, "rency": 5227, "\u0120corner": 5228, "\u0120husband": 5229, "\u0120internet": 5230, "\u0120Aut": 5231, "ems": 5232, "osen": 5233, "\u0120Atl": 5234, "gen": 5235, "\u0120balance": 5236, "62": 5237, "\u0120sounds": 5238, "text": 5239, "\u0120arr": 5240, "oves": 5241, "\u0120millions": 5242, "\u0120radio": 5243, "\u0120satisf": 5244, "\u0120Dam": 5245, "Mr": 5246, "Go": 5247, "Spe": 5248, "\u0120combat": 5249, "rant": 5250, "\u0120Gree": 5251, "\u0120fuel": 5252, "\u0120distance": 5253, "\u0120tests": 5254, "\u0120decre": 5255, "\u0120Er": 5256, "\u0120managed": 5257, "DS": 5258, "\u0120tit": 5259, "\u0120measures": 5260, "\u0120Liber": 5261, "\u0120attend": 5262, "ashed": 5263, "\u0120Jose": 5264, "\u0120Night": 5265, "dit": 5266, "\u0120Nov": 5267, "\u0120End": 5268, "outs": 5269, "\u0120generation": 5270, "\u0120advoc": 5271, "yth": 5272, "\u0120conversation": 5273, "\u0120Sky": 5274, "active": 5275, "cel": 5276, "rier": 5277, "\u0120Frank": 5278, "\u0120gender": 5279, "\u0120concent": 5280, "\u0120carried": 5281, "anda": 5282, "\u0120Virgin": 5283, "\u0120arrived": 5284, "icide": 5285, "aded": 5286, "\u0120failure": 5287, "\u0120minimum": 5288, "lets": 5289, "\u0120worst": 5290, "\u0120keeping": 5291, "\u0120intended": 5292, "\u0120illegal": 5293, "\u0120subsc": 5294, "\u0120determined": 5295, "\u0120trip": 5296, "Yes": 5297, "\u0120raise": 5298, "\u0120~": 5299, "\u0120feels": 5300, "\u0120package": 5301, "\u0120Jo": 5302, "hi": 5303, "2016": 5304, "real": 5305, "\u0120fra": 5306, "\u0120symb": 5307, "Me": 5308, "ucky": 5309, "pret": 5310, "\u0120Kh": 5311, "\u0120Edit": 5312, "\u0120Web": 5313, "emic": 5314, "\u0120Color": 5315, "\u0120justice": 5316, "Int": 5317, "\u0120farm": 5318, "cknow": 5319, "\">": 5320, "eless": 5321, "\u0120reduced": 5322, "\u0120500": 5323, "xx": 5324, "\u0120Rad": 5325, "\u0120Wood": 5326, "\u0120clin": 5327, "\u0120hyp": 5328, "iler": 5329, "ura": 5330, "kins": 5331, "85": 5332, "61": 5333, "\u0120Their": 5334, "\u0120Mary": 5335, "\u0120san": 5336, "\u0120novel": 5337, "\u0120Who": 5338, "\u0120capacity": 5339, "\u0120impossible": 5340, "\u0120plays": 5341, "\u0120minister": 5342, "ijuana": 5343, "icate": 5344, "\u0120Set": 5345, "\u0120fram": 5346, "\u0120ing": 5347, "\u0120communities": 5348, "\u0120FBI": 5349, "ita": 5350, "\u0120bon": 5351, "\u0120strateg": 5352, "\u0120interests": 5353, "lock": 5354, "gers": 5355, "mas": 5356, "\u0120AND": 5357, "\u0120conflict": 5358, "\u0120requirements": 5359, "\u0120sac": 5360, "\u0120operating": 5361, "ini": 5362, "related": 5363, "\u0120committed": 5364, "\u0120relatively": 5365, "\u0120south": 5366, "\u00c2\u00af\u00c2\u00af": 5367, "\u0120afford": 5368, "\u0120identity": 5369, "\u0120decisions": 5370, "\u0120accused": 5371, "place": 5372, "\u0120victory": 5373, "och": 5374, "iat": 5375, "Name": 5376, "Com": 5377, "tion": 5378, "eds": 5379, "\u0120seek": 5380, "\u0120tight": 5381, "\u0120Images": 5382, "\u0120initi": 5383, "\u0120humans": 5384, "\u0120familiar": 5385, "\u0120audience": 5386, "\u0120internal": 5387, "venture": 5388, "\u0120sides": 5389, "\u0120TO": 5390, "\u0120dim": 5391, "\u0120conclud": 5392, "\u0120appoint": 5393, "\u0120enforcement": 5394, "\u0120Jim": 5395, "\u0120Association": 5396, "\u0120circumst": 5397, "\u0120Canadian": 5398, "\u0120joined": 5399, "\u0120differences": 5400, "\u0120Los": 5401, "\u0120protest": 5402, "\u0120twice": 5403, "win": 5404, "\u0120glass": 5405, "arsh": 5406, "\u0120Army": 5407, "\u0120expression": 5408, "\u0120decide": 5409, "\u0120planning": 5410, "ania": 5411, "\u0120handle": 5412, "\u0120Microsoft": 5413, "\u0120Nor": 5414, "\u0120maximum": 5415, "\u0120Rev": 5416, "\u0120sea": 5417, "\u0120eval": 5418, "\u0120helps": 5419, "ref": 5420, "\u0120bound": 5421, "\u0120mouth": 5422, "\u0120standards": 5423, "\u0120clim": 5424, "\u0120Camp": 5425, "\u0120Fox": 5426, "cles": 5427, "\u0120army": 5428, "\u0120Techn": 5429, "acking": 5430, "xy": 5431, "SS": 5432, "\u012042": 5433, "\u0120bug": 5434, "\u0120Ukrain": 5435, "\u0120Max": 5436, "\u0120Jones": 5437, "\u0120Show": 5438, "lo": 5439, "\u0120planet": 5440, "\u012075": 5441, "\u0120winning": 5442, "\u0120faster": 5443, "\u0120spect": 5444, "\u0120broken": 5445, "TR": 5446, "\u0120defined": 5447, "\u0120healthy": 5448, "\u0120competition": 5449, "https": 5450, "\u0120Island": 5451, "\u0120Fe": 5452, "\u0120announce": 5453, "\u0120Cup": 5454, "\u0120Instead": 5455, "\u0120client": 5456, "\u0120possibly": 5457, "section": 5458, "ocket": 5459, "look": 5460, "\u0120finish": 5461, "\u0120crew": 5462, "\u0120reserv": 5463, "\u0120editor": 5464, "\u0120hate": 5465, "\u0120sale": 5466, "\u0120controvers": 5467, "\u0120pages": 5468, "wing": 5469, "\u0120numer": 5470, "\u0120opposition": 5471, "\u01202004": 5472, "\u0120refuge": 5473, "\u0120flight": 5474, "\u0120apart": 5475, "\u0120Lat": 5476, "Americ": 5477, "\u0120Africa": 5478, "\u0120applications": 5479, "\u0120Palest": 5480, "\u0120Bur": 5481, "\u0120gar": 5482, "\u0120Social": 5483, "\u0120upgr": 5484, "\u0120shape": 5485, "\u0120speaking": 5486, "ansion": 5487, "ao": 5488, "\u0120Sn": 5489, "\u0120worry": 5490, "\u0120Britain": 5491, "Please": 5492, "roud": 5493, "\u0120hun": 5494, "\u0120introduced": 5495, "\u0120diet": 5496, "Ind": 5497, "\u0120Second": 5498, "\u0120functions": 5499, "uts": 5500, "\u0120Each": 5501, "\u0120Jeff": 5502, "\u0120stress": 5503, "\u0120accounts": 5504, "\u0120guarant": 5505, "\u0120Ann": 5506, "edia": 5507, "\u0120honest": 5508, "\u0120tree": 5509, "\u0120African": 5510, "\u0120Bush": 5511, "},": 5512, "\u0120sch": 5513, "\u0120Only": 5514, "\u0120fif": 5515, "igan": 5516, "\u0120exercise": 5517, "\u0120Exp": 5518, "\u0120scientists": 5519, "\u0120legislation": 5520, "\u0120Work": 5521, "\u0120Spr": 5522, "\u00c3\u0124": 5523, "\u0120Human": 5524, "\u0120\u00e8": 5525, "\u0120survey": 5526, "\u0120rich": 5527, "rip": 5528, "\u0120maintain": 5529, "\u0120flo": 5530, "\u0120leadership": 5531, "stream": 5532, "\u0120Islamic": 5533, "\u012001": 5534, "\u0120College": 5535, "\u0120magic": 5536, "\u0120Prime": 5537, "\u0120figures": 5538, "2017": 5539, "inder": 5540, "xual": 5541, "\u0120Dead": 5542, "\u0120absolutely": 5543, "\u0120fourth": 5544, "\u0120presented": 5545, "respond": 5546, "rible": 5547, "\u0120alcohol": 5548, "ato": 5549, "\u0120DE": 5550, "porary": 5551, "\u0120grab": 5552, "\u0120vari": 5553, "\u0120quant": 5554, "\u0120Photo": 5555, "\u0120plus": 5556, "rick": 5557, "arks": 5558, "\u0120alternative": 5559, "\u0120pil": 5560, "\u0120approx": 5561, "that": 5562, "\u0120objects": 5563, "\u0120Ro": 5564, "\u0120Android": 5565, "\u0120significantly": 5566, "\u0120Road": 5567, "kay": 5568, "Read": 5569, "avor": 5570, "\u0120acknow": 5571, "\u0120HD": 5572, "\u0120Sing": 5573, "Or": 5574, "\u0120Mont": 5575, "\u0120uns": 5576, "prof": 5577, "\u0120negoti": 5578, "\u0120Arch": 5579, "iki": 5580, "\u0120television": 5581, "\u0120Jewish": 5582, "\u0120committee": 5583, "\u0120motor": 5584, "\u0120appearance": 5585, "\u0120sitting": 5586, "\u0120strike": 5587, "\u0120Down": 5588, "comp": 5589, "\u0120Hist": 5590, "\u0120fold": 5591, "acement": 5592, "\u0120Louis": 5593, "\u0120belong": 5594, "\u0120\u00e2\u0122\u00a2": 5595, "\u0120mort": 5596, "\u0120prepared": 5597, "\u012064": 5598, "\u0120Master": 5599, "\u0120indeed": 5600, "\u0120Den": 5601, "\u0120rent": 5602, "TA": 5603, "ourney": 5604, "arc": 5605, "Su": 5606, "97": 5607, "\u0120advice": 5608, "\u0120changing": 5609, "\u0120listed": 5610, "\u0120launched": 5611, "isation": 5612, "\u0120Peter": 5613, "ishes": 5614, "\u0120lived": 5615, "\u0120Mel": 5616, "\u0120Supreme": 5617, "\u0120Federal": 5618, "\u0120);": 5619, "ructure": 5620, "\u0120sets": 5621, "\u0120philos": 5622, "uous": 5623, "\u0120\u00c2\u0142": 5624, "\u0120applied": 5625, "\u0120NOT": 5626, "\u0120housing": 5627, "\u0120Mount": 5628, "\u0120odd": 5629, "\u0120sust": 5630, "DA": 5631, "fficient": 5632, "\u0120?": 5633, "olved": 5634, "\u0120powers": 5635, "\u0120thr": 5636, "\u0120remaining": 5637, "\u0120Water": 5638, "LC": 5639, "\u0120causes": 5640, "\u00e3\u0123\u00ae": 5641, "\u0120manner": 5642, "ads": 5643, "\u0120suggests": 5644, "\u0120ends": 5645, "standing": 5646, "fig": 5647, "\u0120Dun": 5648, "idth": 5649, "\u0120gay": 5650, "\u0120termin": 5651, "\u0120Angeles": 5652, "MS": 5653, "\u0120scientific": 5654, "\u0120coal": 5655, "apers": 5656, "bar": 5657, "\u0120Thomas": 5658, "\u0120sym": 5659, "\u0120Run": 5660, "this": 5661, "PC": 5662, "igrants": 5663, "\u0120minute": 5664, "\u0120District": 5665, "cellent": 5666, "\u0120leaves": 5667, "\u0120completed": 5668, "amin": 5669, "\u0120focused": 5670, "\u0120monitor": 5671, "\u0120vehicles": 5672, "MA": 5673, "\u0120Mass": 5674, "\u0120Grand": 5675, "\u0120affected": 5676, "itutional": 5677, "\u0120construct": 5678, "\u0120follows": 5679, "\u0120ton": 5680, "reens": 5681, "\u0120homes": 5682, "\u0120Ext": 5683, "\u0120Level": 5684, "rast": 5685, "\u0120Ir": 5686, "\u0120elim": 5687, "\u0120largely": 5688, "\u0120Joe": 5689, "\u0120votes": 5690, "alls": 5691, "\u0120businesses": 5692, "\u0120Foundation": 5693, "\u0120Central": 5694, "\u0120yards": 5695, "\u0120materials": 5696, "ulner": 5697, "\u0120guide": 5698, "\u0120closer": 5699, "ums": 5700, "\u0120sports": 5701, "eder": 5702, "Just": 5703, "\u0120taxes": 5704, "84": 5705, "\u0120Old": 5706, "\u0120decade": 5707, "ola": 5708, "\u0120vir": 5709, "\u0120dropped": 5710, "\u0120delay": 5711, "itect": 5712, "\u0120secure": 5713, "stein": 5714, "level": 5715, "\u0120treated": 5716, "\u0120filed": 5717, "aine": 5718, "\u0120van": 5719, "\u0120mir": 5720, "\u0120column": 5721, "icted": 5722, "eper": 5723, "\u0120rot": 5724, "\u0120consult": 5725, "\u0120entry": 5726, "\u0120marijuana": 5727, "\u0120Dou": 5728, "\u0120apparently": 5729, "oking": 5730, "clusive": 5731, "\u0120increases": 5732, "ano": 5733, "\u0120specifically": 5734, "\u0120tele": 5735, "ensions": 5736, "\u0120religion": 5737, "abilities": 5738, "\u0120frame": 5739, "\u0120Note": 5740, "\u0120Lee": 5741, "\u0120helping": 5742, "\u0120edge": 5743, "oston": 5744, "\u0120organizations": 5745, "\u00c3\u0125": 5746, "\u0120Both": 5747, "hips": 5748, "\u0120bigger": 5749, "\u0120boost": 5750, "\u0120Stand": 5751, "\u0120row": 5752, "uls": 5753, "abase": 5754, "\u0120rid": 5755, "Let": 5756, "aren": 5757, "rave": 5758, "\u0120stret": 5759, "PD": 5760, "\u0120vision": 5761, "\u0120wearing": 5762, "\u0120appreci": 5763, "\u0120award": 5764, "\u0120Use": 5765, "\u0120factor": 5766, "war": 5767, "ulations": 5768, ")(": 5769, "\u0120god": 5770, "\u0120territ": 5771, "\u0120param": 5772, "asts": 5773, "87": 5774, "\u0120enemies": 5775, "\u0120Games": 5776, "FF": 5777, "\u0120accident": 5778, "Well": 5779, "\u0120Martin": 5780, "TER": 5781, "\u0120ath": 5782, "\u0120Hell": 5783, "\u0120forg": 5784, "\u0120veter": 5785, "\u0120Medic": 5786, "free": 5787, "\u0120stars": 5788, "\u0120expensive": 5789, "\u0120acad": 5790, "rawn": 5791, "\u0120Whe": 5792, "\u0120lock": 5793, "\u0120format": 5794, "\u0120soldiers": 5795, "sm": 5796, "\u0120agent": 5797, "\u0120responsibility": 5798, "ora": 5799, "\u0120Science": 5800, "\u0120rapid": 5801, "\u0120tough": 5802, "\u0120Jesus": 5803, "\u0120believes": 5804, "ML": 5805, "\u0120wear": 5806, "lete": 5807, "\u00c3\u0125\u00c3\u0124": 5808, "\u0120Dri": 5809, "\u0120commission": 5810, "\u0120Bob": 5811, "Oh": 5812, "aped": 5813, "\u0120warm": 5814, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 5815, "\u01202003": 5816, "ortion": 5817, "\u0120hasn": 5818, "uster": 5819, "\u0120univers": 5820, "\u0120Ill": 5821, "\u0120king": 5822, "ologies": 5823, "94": 5824, "\u0120Tem": 5825, "\u0120Mos": 5826, "\u0120patient": 5827, "\u0120Mexico": 5828, "cean": 5829, "\u0120Death": 5830, "\u0120Sanders": 5831, "you": 5832, "\u0120Cast": 5833, "\u0120Company": 5834, "pty": 5835, "\u0120happening": 5836, "FP": 5837, "\u0120Battle": 5838, "\u0120bought": 5839, "Am": 5840, "Mod": 5841, "Us": 5842, "uters": 5843, "\u0120Cre": 5844, "\u0120Those": 5845, "\u012044": 5846, "iser": 5847, "\u0120soul": 5848, "\u0120Top": 5849, "\u0120Harry": 5850, "\u0120Aw": 5851, "\u0120seat": 5852, "ffee": 5853, "\u0120revolution": 5854, "\u0120(\"": 5855, "\u0120During": 5856, "ette": 5857, "\u0120ring": 5858, "\u0120offensive": 5859, "\u0120returns": 5860, "\u0120videos": 5861, "\u0120discl": 5862, "\u0120famous": 5863, "enced": 5864, "\u0120Sign": 5865, "\u0120River": 5866, "\u0120300": 5867, "PM": 5868, "\u0120Bus": 5869, "\u0120CH": 5870, "\u0120candidates": 5871, "arden": 5872, "\u0120percentage": 5873, "\u0120visual": 5874, "\u0120thank": 5875, "\u0120trouble": 5876, "nergy": 5877, "\u01202001": 5878, "\u0120prove": 5879, "ashion": 5880, "\u0120enh": 5881, "\u0120Long": 5882, "UM": 5883, "\u0120connected": 5884, "\u0120possibility": 5885, "Over": 5886, "\u0120expert": 5887, "\u0120library": 5888, "arts": 5889, "\u0120Director": 5890, "\u0120fellow": 5891, "92": 5892, "irty": 5893, "\u0120dry": 5894, "\u0120signs": 5895, "\u0120Love": 5896, "\u0120quiet": 5897, "foot": 5898, "\u0120pure": 5899, "\u0120Hun": 5900, "\u0120filled": 5901, "phas": 5902, "\u0120Elect": 5903, "endment": 5904, "\u0120Expl": 5905, "\u0120unable": 5906, "ns": 5907, "mo": 5908, "\u0120vast": 5909, "obe": 5910, "\u0120identify": 5911, "apping": 5912, "\u0120Carolina": 5913, "gress": 5914, "\u0120prote": 5915, "\u0120fish": 5916, "\u0120circumstances": 5917, "razy": 5918, "\u0120Phot": 5919, "\u0120bodies": 5920, "\u0120Mur": 5921, "\u0120developing": 5922, "\u0120AR": 5923, "\u0120experienced": 5924, "\u0120substant": 5925, "\u0120Board": 5926, "esome": 5927, "\u0120domestic": 5928, "\u0120combined": 5929, "\u0120Put": 5930, "\u0120chemical": 5931, "\u0120Child": 5932, "\u0120pool": 5933, "\u0120Cy": 5934, "\u0120egg": 5935, "cons": 5936, "sters": 5937, "\u0120hurt": 5938, "\u0120markets": 5939, "\u0120conservative": 5940, "\u0120supporters": 5941, "\u0120agencies": 5942, "idel": 5943, "Ob": 5944, "urb": 5945, "\u012043": 5946, "\u0120Defense": 5947, "ye": 5948, "\u0120Ap": 5949, "dule": 5950, "\u0120temperature": 5951, "\u0120conducted": 5952, "\u0120Chief": 5953, "\u0120pulled": 5954, "\u0120fol": 5955, "Last": 5956, "onto": 5957, "osis": 5958, "VER": 5959, "Des": 5960, "\u0120Pan": 5961, "First": 5962, "\u0120advance": 5963, "\u0120license": 5964, "rors": 5965, "\u0120Jon": 5966, "\u0120imagine": 5967, "\u0120hell": 5968, "\u0120fixed": 5969, "\u0120incor": 5970, "osite": 5971, "\u0120Log": 5972, "icken": 5973, "]:": 5974, "\u0120surprise": 5975, "hab": 5976, "\u0120craft": 5977, "olt": 5978, "\u0120Jul": 5979, "\u0120dial": 5980, "\u0120relevant": 5981, "\u0120entered": 5982, "\u0120leads": 5983, "\u0120AD": 5984, "\u0120Clean": 5985, "\u0120pictures": 5986, "essor": 5987, "\u0120alt": 5988, "\u0120paying": 5989, "Per": 5990, "\u0120Market": 5991, "\u0120updates": 5992, "amily": 5993, "\u0120Type": 5994, "\u0120Home": 5995, "\u012055": 5996, "sembly": 5997, "rome": 5998, "83": 5999, "\u0120greatest": 6000, "\u0120height": 6001, "\u0120heav": 6002, "aints": 6003, "\u0120listen": 6004, "aser": 6005, "\u0120SH": 6006, "\u0120capable": 6007, "acle": 6008, "\u0120perspect": 6009, "inating": 6010, "\u0120offering": 6011, "rypt": 6012, "\u0120Develop": 6013, "abin": 6014, "rc": 6015, "\u0120bright": 6016, "alty": 6017, "arrow": 6018, "\u0120suppl": 6019, "inding": 6020, "acked": 6021, "gypt": 6022, "\u0120Another": 6023, "pg": 6024, "\u0120Virginia": 6025, "\u0120Lu": 6026, "\u0120planned": 6027, "\u0120pit": 6028, "\u0120sweet": 6029, "Type": 6030, "\u0120Di": 6031, "\u0120typically": 6032, "\u0120Francisco": 6033, "\u0120prospect": 6034, "\u0120Dan": 6035, "\u0120teen": 6036, "rees": 6037, "\u0120sched": 6038, "\u0120hol": 6039, "\u0120scr": 6040, "\u0120lots": 6041, "life": 6042, "\u0120newsp": 6043, "\u0120forget": 6044, "\u0120None": 6045, "\u0120Middle": 6046, "\u0120Ryan": 6047, "edd": 6048, "\u0120severe": 6049, "\u0120suit": 6050, "ller": 6051, "93": 6052, "\u0120correspond": 6053, "\u0120explos": 6054, "uations": 6055, "\u0120flag": 6056, "game": 6057, "rid": 6058, "\u0120prin": 6059, "\u0120Data": 6060, "\u0120deploy": 6061, "\u0120Enter": 6062, "suit": 6063, "ghan": 6064, "\u0120Men": 6065, "\u0120thoughts": 6066, "\u0120matters": 6067, "\u0120adapt": 6068, "\u0120Ari": 6069, "\u0120fill": 6070, "\u0120forth": 6071, "\u0120sam": 6072, "\u012041": 6073, "\u0120payment": 6074, "\u0120Hor": 6075, "\u0120spring": 6076, "duc": 6077, "\u0120losing": 6078, "\u0120bringing": 6079, "FO": 6080, "ala": 6081, "\u0120distribution": 6082, "hered": 6083, "bour": 6084, "\u0120Israeli": 6085, "oma": 6086, "\u0120combination": 6087, "\u0120plenty": 6088, "VE": 6089, "Can": 6090, "\u0120Haw": 6091, "\u0120perman": 6092, "\u0120Special": 6093, "\u0120tow": 6094, "\u0120seeking": 6095, "\u0120examples": 6096, "\u0120classes": 6097, "cr": 6098, "\u0120beer": 6099, "\u0120moves": 6100, "\u0120IP": 6101, "\u0120Kn": 6102, "\u0120panel": 6103, "Even": 6104, "\u0120properly": 6105, "\u0120ris": 6106, "\u0120plug": 6107, "\u0120estimated": 6108, "Every": 6109, "\u0120defensive": 6110, "agraph": 6111, "\u0120pregn": 6112, "\u0120instit": 6113, "\u0120Vict": 6114, "\u0120volume": 6115, "\u0120positions": 6116, "\u0120links": 6117, "\u0120Program": 6118, "\u0120Week": 6119, "agues": 6120, "\u0120transform": 6121, "ker": 6122, "\u0120CEO": 6123, "\u0120cas": 6124, "\u0120opponent": 6125, "\u0120tweet": 6126, "\u0120Code": 6127, "\u0120shop": 6128, "\u0120fly": 6129, "\u0120talks": 6130, "\u0120bag": 6131, "Phone": 6132, "\u0120aid": 6133, "\u0120plants": 6134, "\u012065": 6135, "\u0120attorney": 6136, "arters": 6137, "quest": 6138, "\u0120Magic": 6139, "\u0120begins": 6140, "\u0120myster": 6141, "\u0120environmental": 6142, "\u0120storage": 6143, "NN": 6144, "\u0120marg": 6145, "\u0120ske": 6146, "\u0120metal": 6147, "elly": 6148, "\u0120ordered": 6149, "\u0120remained": 6150, "\u0120loved": 6151, "\u0120prompt": 6152, "\u0120updated": 6153, "\u0120experts": 6154, "\u0120walking": 6155, "\u0120ancient": 6156, "\u0120performed": 6157, "ATE": 6158, "\u0120neither": 6159, "iency": 6160, "\u0120manufacture": 6161, "\u0120Pak": 6162, "\u0120selected": 6163, "\u0120mine": 6164, "\u0120ultimately": 6165, "\u0120explan": 6166, "\u0120label": 6167, "\u0120Services": 6168, "ributed": 6169, "Trump": 6170, "\u0120syn": 6171, "\u0120Ult": 6172, "SC": 6173, "\u0120meat": 6174, "\u0120giant": 6175, "\u0120Wars": 6176, "\u0120ON": 6177, "\u0120adm": 6178, "\u0120interpret": 6179, "\u0120evening": 6180, "\u0120evil": 6181, "\u0120Boston": 6182, "\u0120Wild": 6183, "\u0120\u00c3": 6184, "\u0120Bitcoin": 6185, "\u0120Amazon": 6186, "Dr": 6187, "\u0120Information": 6188, "\u0120obviously": 6189, "\u0120advanced": 6190, "Photo": 6191, "olar": 6192, "\u0120weather": 6193, "\u0120symbol": 6194, "\u0120sole": 6195, "\u0120potentially": 6196, "oster": 6197, "\u0120originally": 6198, "mun": 6199, "300": 6200, "aze": 6201, "essions": 6202, "\u0120deck": 6203, "\u0120stood": 6204, "\u0120youth": 6205, "\u0120Bern": 6206, "Rep": 6207, "\u0120Test": 6208, "\u0120basically": 6209, "otic": 6210, "\u0120involve": 6211, "olit": 6212, "lyn": 6213, "See": 6214, "\u0120aircraft": 6215, "\u0120confirm": 6216, "EW": 6217, "\u0120messages": 6218, "\u0120Richard": 6219, "\u0120kit": 6220, "\u0120prohib": 6221, "\u0120vulner": 6222, "isters": 6223, "\u0120existence": 6224, "\u0120turning": 6225, "\u0120SP": 6226, "\u0120desire": 6227, "\u0120flat": 6228, "\u0120ment": 6229, "season": 6230, "anges": 6231, "\u0120neighborhood": 6232, "\u0120Lake": 6233, "ATION": 6234, "\u0120pointed": 6235, "bur": 6236, "\u0120innov": 6237, "ucks": 6238, "UL": 6239, "\u0120professor": 6240, "\u0120expressed": 6241, "AB": 6242, "icious": 6243, "\u01202002": 6244, "\u0120Dev": 6245, "\u0120session": 6246, "\u0120bare": 6247, "sen": 6248, "\u0120diss": 6249, "\u0120Cath": 6250, "\u0120Pass": 6251, "\u0120Point": 6252, "\u0120doctor": 6253, "orrow": 6254, "ailed": 6255, "\u0120Rub": 6256, "\u0120DC": 6257, "\u0120Charl": 6258, "person": 6259, "\u0120writer": 6260, "ighters": 6261, "ureau": 6262, "\u0120oblig": 6263, "\u0120recorded": 6264, "\u0120broke": 6265, "\u0120orders": 6266, "ilty": 6267, "\u0120motion": 6268, "inity": 6269, "law": 6270, "adium": 6271, "\u0120immigration": 6272, "\u0120contrast": 6273, "\u0120batt": 6274, "\u0120excellent": 6275, "\u0120technical": 6276, "ami": 6277, "\u0120tun": 6278, "\u0120cloud": 6279, "\u0120Year": 6280, "geon": 6281, "\u0120creation": 6282, "\u0120strange": 6283, "\u0120auth": 6284, "\u0120fort": 6285, "born": 6286, "\u0120extent": 6287, "\u0120Today": 6288, "\u0120Club": 6289, "\u0120rain": 6290, "\u0120sample": 6291, "\u0120accepted": 6292, "\u0120tact": 6293, "\u0120fired": 6294, "\u0120Son": 6295, "\u0120stands": 6296, "\u0120boot": 6297, "\u012047": 6298, "\u0120statements": 6299, "\u0120versions": 6300, "\u0120selling": 6301, "ounded": 6302, "\u01201990": 6303, "\u0120weren": 6304, "\u0120Watch": 6305, "\u0120experiment": 6306, "Post": 6307, "\u0120retail": 6308, "uled": 6309, "Inst": 6310, "unte": 6311, "\u00e3\u0125\u00bc": 6312, "\u0120depart": 6313, "\u0120bond": 6314, "ivery": 6315, "ompl": 6316, "\u0120reaction": 6317, "\u0120Syrian": 6318, "\u0120Pac": 6319, "apped": 6320, "aniel": 6321, "DP": 6322, "\u0120resolution": 6323, "\u0120react": 6324, "\u0120approved": 6325, "onom": 6326, "mond": 6327, "\u0120Offic": 6328, "---": 6329, "\u0120replace": 6330, "\u0120tack": 6331, "\u0120sport": 6332, "\u0120chain": 6333, "\u0120emergency": 6334, "rad": 6335, "\u0120Palestin": 6336, "\u012046": 6337, "\u0120automatically": 6338, "\u0120route": 6339, "\u0120pal": 6340, "\u0120banks": 6341, "\u0120Paris": 6342, "\u0120Media": 6343, "road": 6344, "icing": 6345, "ixt": 6346, "isted": 6347, "\u0120grew": 6348, "\u0120coord": 6349, "\u0120Where": 6350, "omin": 6351, "\u0120subs": 6352, "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 6353, "\u0120\u00c2\u00b1": 6354, "\u0120corporate": 6355, "\u0120selection": 6356, "noon": 6357, "\u0120Report": 6358, "cs": 6359, "cluding": 6360, "orders": 6361, "anche": 6362, "\u0120Its": 6363, "\u0120slowly": 6364, "\u0120Egypt": 6365, "\u0120Acc": 6366, "\u0120colle": 6367, "iques": 6368, "EX": 6369, "\u0120attempts": 6370, "url": 6371, "\u0120Cross": 6372, "\u0120findings": 6373, "\u0120SC": 6374, "\u0120OR": 6375, "\u0120index": 6376, "ensity": 6377, "\u0120Way": 6378, "\u0120Land": 6379, "\u0120shock": 6380, "dis": 6381, "\u0120dynam": 6382, "\u0120cart": 6383, "mosp": 6384, "Since": 6385, "iest": 6386, "\u0120Boy": 6387, "\u0120storm": 6388, "\u0120Contin": 6389, "2013": 6390, "hew": 6391, "ilit": 6392, "\u0120essential": 6393, "iquid": 6394, "Other": 6395, "ivered": 6396, "\u0120reasonable": 6397, "Act": 6398, "\u0120subsequ": 6399, "\u0120Pack": 6400, "\u0120Fort": 6401, "\u0120considering": 6402, "\u0120university": 6403, "log": 6404, "\u0120married": 6405, "\u0120illust": 6406, "\u0120True": 6407, "\u00a3\u0131": 6408, "\u0120numerous": 6409, "rastructure": 6410, "\u0120seriously": 6411, "\u0120referred": 6412, "ua": 6413, "\u0120consistent": 6414, "onna": 6415, "\u0120Real": 6416, "ruption": 6417, "ciples": 6418, "\u0120facts": 6419, "91": 6420, "otes": 6421, "erg": 6422, "Then": 6423, "\u0120accompl": 6424, "Note": 6425, "\u0120revenue": 6426, "\u0120passing": 6427, "\u0120mal": 6428, "een": 6429, "\u0120Yet": 6430, "\u0120gather": 6431, "terday": 6432, "ework": 6433, "\u0120Author": 6434, "Pe": 6435, "\u0120optim": 6436, "\u0120rub": 6437, "\u0120\u00e8\u00a3\u0131": 6438, "\u0120unknown": 6439, "stone": 6440, "\u0120union": 6441, "olve": 6442, "\u0120opportunities": 6443, "\u0120browser": 6444, "\u0120Wal": 6445, "\u0120Cost": 6446, "\u0120reporting": 6447, "sts": 6448, "pet": 6449, "\u0120sand": 6450, "\u0120suddenly": 6451, "\u0120surprising": 6452, "\u0120VR": 6453, "\u0120somewhat": 6454, "\u0120Bas": 6455, "ulture": 6456, "izz": 6457, "\u0120CD": 6458, "\u0120challenges": 6459, "\u0120settings": 6460, "\u0120experiences": 6461, "\u0120Full": 6462, "\u0120cann": 6463, "\u0120receiving": 6464, "EST": 6465, "\u0120joint": 6466, "\u0120cultural": 6467, "\u0120ast": 6468, "82": 6469, "astern": 6470, "ceived": 6471, "\u0120Cru": 6472, "\u0120bull": 6473, "pired": 6474, "amm": 6475, "\u0120facing": 6476, "power": 6477, "\u0120boss": 6478, "\u0120Hol": 6479, "\u0120instr": 6480, "\u0120increasingly": 6481, "\u0120shift": 6482, "\u0120streets": 6483, "\u0120Williams": 6484, "abb": 6485, "\u0120lie": 6486, "\u0120laugh": 6487, "\u0120Ca": 6488, "PL": 6489, "\u0120adults": 6490, "\u0120customer": 6491, "\u0120obtained": 6492, "\u0120supporting": 6493, "html": 6494, "fire": 6495, "\u0120detailed": 6496, "\u0120picked": 6497, "\u0120Right": 6498, "lder": 6499, "EE": 6500, "stood": 6501, "\u0120Kim": 6502, "\u0120wire": 6503, "\u0120sight": 6504, "\u0120developers": 6505, "\u0120persons": 6506, "\u0120sad": 6507, "\u0120cup": 6508, "\u0120warning": 6509, "\u0120boys": 6510, "long": 6511, "\u0120bird": 6512, "fo": 6513, "\u0120wal": 6514, "\u0120observed": 6515, "\u0120zone": 6516, "iveness": 6517, "\u0120channel": 6518, "cript": 6519, "\u0120refused": 6520, "\u0120Again": 6521, "\u0120suc": 6522, "\u0120spokesman": 6523, "\u0120Ref": 6524, "rite": 6525, "ouston": 6526, "\u00e3\u0125\u00b3": 6527, "\u0120Sher": 6528, "\u0120acts": 6529, "\u0120Name": 6530, "\u0120struggle": 6531, "arry": 6532, "ometimes": 6533, "\u0120discrim": 6534, "HT": 6535, "\u0120category": 6536, "\u0120realize": 6537, "\u0120employee": 6538, "\u0120Afghan": 6539, "enger": 6540, "\u0120guns": 6541, "\u0120Steve": 6542, "\u0120Mot": 6543, "\u0120Ol": 6544, "oked": 6545, "\u0120thick": 6546, "\u0120fairly": 6547, "illy": 6548, "\u0120surve": 6549, "\u0120Mat": 6550, "weight": 6551, "\u00e2\u0136": 6552, "\u0120troops": 6553, "\u0120agents": 6554, "\u0120battery": 6555, "\u0120motiv": 6556, "\u00c3\u00a1": 6557, "Sec": 6558, "den": 6559, "overy": 6560, "LS": 6561, "\u0120flu": 6562, "\u0120confident": 6563, "\u0120Oper": 6564, "\u0120empty": 6565, "\u0120phen": 6566, "\u0120sector": 6567, "\u0120excited": 6568, "\u0120remote": 6569, "aph": 6570, "oen": 6571, "\u0120destroyed": 6572, "\u0120moral": 6573, "\u0120HP": 6574, "\u0120Ron": 6575, "\u0120dress": 6576, "\u0120Bat": 6577, "\u0120lit": 6578, "\u0120MS": 6579, "\u0120af": 6580, "HL": 6581, "rum": 6582, "isms": 6583, "\u0120shouldn": 6584, "\u0120sympt": 6585, "\u0120Toronto": 6586, "hetic": 6587, "\u0120carbon": 6588, "\u0120installed": 6589, "\u0120violent": 6590, "\u0120solar": 6591, "ja": 6592, "\u0120practices": 6593, "\u0120ride": 6594, "\u0120Penn": 6595, "\u0120improved": 6596, "\u0120audio": 6597, "\u0120behavi": 6598, "\u0120PS": 6599, "\u0120eating": 6600, "Data": 6601, "\u0120Review": 6602, "pass": 6603, "claim": 6604, "uated": 6605, "angers": 6606, "chen": 6607, "\u0120properties": 6608, "\u0120anywhere": 6609, "Another": 6610, "\u0120blow": 6611, "\u0120Jackson": 6612, "\u0120proud": 6613, "\u0120plane": 6614, "lines": 6615, "\u0120square": 6616, "\u0120proof": 6617, "ansas": 6618, "\u0120talked": 6619, "makers": 6620, "\u0120sister": 6621, "\u0120holds": 6622, "\u0120resident": 6623, "\u0120==": 6624, "\u0120resistance": 6625, "\u0120split": 6626, "\u0120prosecut": 6627, "\u0120confidence": 6628, "resents": 6629, "\u0120cuts": 6630, "\u0120exception": 6631, "\u0120zero": 6632, "Getty": 6633, "\u0120copyright": 6634, "\u0120totally": 6635, "ormal": 6636, "ifications": 6637, "\u0120Australian": 6638, "\u0120sick": 6639, "\u0120150": 6640, "\u0120household": 6641, "\u0120fees": 6642, "\u0120drivers": 6643, "ogen": 6644, "\u0120NY": 6645, "\u0120necessarily": 6646, "\u0120regulations": 6647, "earing": 6648, "sl": 6649, "\u0120perspective": 6650, "care": 6651, "icial": 6652, "His": 6653, "\u0120escape": 6654, "\u0120surprised": 6655, "\u0120Van": 6656, "urrent": 6657, "\u0120vac": 6658, "81": 6659, "\u0120Thus": 6660, "\u0120emphas": 6661, "\u0120Champions": 6662, "\u0120Ice": 6663, "\u0120narr": 6664, "\u0120heads": 6665, "\u0120causing": 6666, "bel": 6667, "fortunately": 6668, "\u0120Ma": 6669, "\u0120targets": 6670, "cipl": 6671, "\u0120afternoon": 6672, "\u0120adds": 6673, "\u0120Maybe": 6674, "\u0120Four": 6675, "essed": 6676, "plete": 6677, "\u0120usual": 6678, "cho": 6679, "ingu": 6680, "\u0120withd": 6681, "\u0120Energy": 6682, "\u0120Econom": 6683, "OO": 6684, "\u0120articles": 6685, "\u0120injured": 6686, "\u0120manage": 6687, "\u0120explains": 6688, "\u0120diagn": 6689, "Rec": 6690, "atures": 6691, "\u0120linked": 6692, "\u0120discussed": 6693, "\u0120explo": 6694, "\u0120occasion": 6695, "athan": 6696, "\u0120opposite": 6697, "\u0120faces": 6698, "\u0120denied": 6699, "\u0120Knight": 6700, "\u0120nut": 6701, "\u0120approximately": 6702, "\u0120disappoint": 6703, "onymous": 6704, "\u0120Best": 6705, "\u0120Lo": 6706, "\u0120Hy": 6707, "\u0120Aff": 6708, "\u0120voting": 6709, "anwhile": 6710, "\u0120III": 6711, "\u0120institutions": 6712, "agram": 6713, "\u0120Daily": 6714, "\u0120drag": 6715, "\u0120nearby": 6716, "\u0120guilty": 6717, "\u0120conver": 6718, "Pre": 6719, "ship": 6720, "\u0120reward": 6721, "\u0120philosoph": 6722, "\u0120SS": 6723, "ugh": 6724, "\u0120apps": 6725, "friend": 6726, "\u0120upper": 6727, "\u0120advert": 6728, "\u0120snow": 6729, "\u0120frust": 6730, "\u0120ourselves": 6731, "Fr": 6732, "\u0120Die": 6733, "ampion": 6734, "\u0120dismiss": 6735, "\u0120cere": 6736, "\u0120signal": 6737, "from": 6738, "\u0120).": 6739, "\u012052": 6740, "\u0120crimes": 6741, "itors": 6742, "estival": 6743, "useum": 6744, "\u0120council": 6745, "\u0120Saud": 6746, "May": 6747, "\u0120Gun": 6748, "ician": 6749, "ether": 6750, "\u0120sufficient": 6751, "\u0120Hen": 6752, "sole": 6753, "\u0120historical": 6754, "\u0120Far": 6755, "\u0120Turn": 6756, "\u0120pin": 6757, "\u0120succeed": 6758, "mat": 6759, "lymp": 6760, "\u0120tradition": 6761, "\u0120Ok": 6762, "\u0120cro": 6763, "\u0120description": 6764, "alle": 6765, "\u0120sky": 6766, "Te": 6767, "\u0120widely": 6768, "\u0120wave": 6769, "\u0120definition": 6770, "\u0120Jews": 6771, "\u0120cycle": 6772, "\u0120refere": 6773, "\u0120brings": 6774, "usal": 6775, "\u0120alive": 6776, "\u0120frequently": 6777, "\u0120intention": 6778, "\u0120Control": 6779, "lv": 6780, "ystem": 6781, "\u0120privacy": 6782, "gent": 6783, "rence": 6784, "\u0120Quest": 6785, "\u0120Christmas": 6786, "\u0120rail": 6787, "\u0120cooper": 6788, "\u0120tested": 6789, "\u0120Capt": 6790, "asks": 6791, "\u0120comfortable": 6792, "\u0120delivered": 6793, "scape": 6794, "\u0120depth": 6795, "\u0120GOP": 6796, "\u0120writes": 6797, "\u0120assets": 6798, "\u0120sav": 6799, "iments": 6800, "\u0120transition": 6801, "\u0120artist": 6802, "\u0120Look": 6803, "\u0120lob": 6804, "\u0120components": 6805, "arity": 6806, "\u0120walked": 6807, "\u0120root": 6808, "\u0120participants": 6809, "\u0120noticed": 6810, "\u0120resc": 6811, "\u0120nav": 6812, "\u0120Administ": 6813, "da": 6814, "utral": 6815, "plate": 6816, "\u0120importance": 6817, "\u0120assert": 6818, "iously": 6819, "cription": 6820, "\u0120injuries": 6821, "\u0120Check": 6822, "\u0120registered": 6823, "\u0120intent": 6824, "\u0120missed": 6825, "ographic": 6826, "\u0120sentence": 6827, "ounter": 6828, "\u0120assistance": 6829, "evin": 6830, "\u0120database": 6831, "\u0120buildings": 6832, "\u0120classic": 6833, "\u0120thinks": 6834, "\u0120Ohio": 6835, "Pr": 6836, "ugg": 6837, "\u0120fee": 6838, "pan": 6839, "\u0120effectively": 6840, "\u0120facility": 6841, "\u0120bear": 6842, "\u0120chapter": 6843, "\u0120dogs": 6844, "\u0120Columb": 6845, "\u0120latter": 6846, "itial": 6847, "\u0120admitted": 6848, "TV": 6849, "\u0120Georg": 6850, "\u0120posts": 6851, "\\\\": 6852, "\u0120lawyer": 6853, "\u0120equival": 6854, "\u0120mand": 6855, "\u0120controlled": 6856, "\u0120Walk": 6857, "\u0120Andrew": 6858, "\u0120menu": 6859, "amental": 6860, "\u0120protected": 6861, "va": 6862, "\u0120administr": 6863, "oral": 6864, "\u0120rein": 6865, "\u0120Sar": 6866, "\u0120amounts": 6867, "\u0120native": 6868, "\u0120Moon": 6869, "\u0120represents": 6870, "\u0120abandon": 6871, "\u0120carrying": 6872, "\u0120tank": 6873, "mary": 6874, "\u0120declared": 6875, "Tube": 6876, "\u0120hat": 6877, "\u0120punish": 6878, "ellect": 6879, "mes": 6880, "\u0120universe": 6881, "\u0120Rod": 6882, "phy": 6883, "\u0120infrastructure": 6884, "\u012051": 6885, "\u0120opposed": 6886, "ownt": 6887, "ca": 6888, "\u0120Make": 6889, "\u0120hardware": 6890, "\u0120coffee": 6891, "Rel": 6892, "bal": 6893, "world": 6894, "\u0120Saf": 6895, "\u0120Sea": 6896, "inals": 6897, "\u0120owned": 6898, "\u0120hall": 6899, "ersion": 6900, "\u0120describe": 6901, "\u0120Pot": 6902, "\u0120portion": 6903, "\u0120atmosp": 6904, "\u0120governments": 6905, "\u0120depending": 6906, "\u0120offense": 6907, "\u0120trick": 6908, "awa": 6909, "\u0120Line": 6910, "\u0120Vis": 6911, "\u0120Hard": 6912, "\u0120Orig": 6913, "\u0120Click": 6914, "\u0120desk": 6915, "\u0120Valley": 6916, "\u0120Sov": 6917, "\u0120movies": 6918, "\u0120remark": 6919, "\u0120mail": 6920, "\u0120conscious": 6921, "\u0120ruling": 6922, "\u0120Rights": 6923, "\u0120medic": 6924, "hent": 6925, "\u0120Women": 6926, "><": 6927, "\u0120replaced": 6928, "\u0120Prem": 6929, "\u0120Thanks": 6930, "\u0120renew": 6931, "\u0120Ball": 6932, "iform": 6933, "\u0120shots": 6934, "Comm": 6935, "\u0120armed": 6936, "\u0120constant": 6937, "\u0120taste": 6938, "\u0120realized": 6939, "\u0120buff": 6940, "\u0120mo": 6941, "\u0120efficient": 6942, "Most": 6943, "oration": 6944, "ifies": 6945, "\u0120communication": 6946, "\u0120flood": 6947, "\u0120consequences": 6948, "\u0120anyway": 6949, "igg": 6950, "\u0120GM": 6951, "\u0120Thank": 6952, "\u0120iron": 6953, "\u0120evolution": 6954, "\u0120Cop": 6955, "twitter": 6956, "\u012095": 6957, "\u0120relationships": 6958, "adel": 6959, "\u0120Young": 6960, "\u0120proposal": 6961, "ayers": 6962, "uilding": 6963, "\u0120Hot": 6964, "ORE": 6965, "cos": 6966, "\u0120collabor": 6967, "PG": 6968, "axy": 6969, "\u0120knowing": 6970, "\u0120supports": 6971, "owed": 6972, "\u0120controls": 6973, "\u0120merely": 6974, "umer": 6975, "\u0120athlet": 6976, "\u0120fashion": 6977, "path": 6978, "\u0120gift": 6979, "\u0120era": 6980, "AND": 6981, "\u0120kinds": 6982, "\u0120Korean": 6983, "\u0120legit": 6984, "ulous": 6985, "\u0120essentially": 6986, "\u0120therap": 6987, "nic": 6988, "\u0120suffered": 6989, "\u0120hur": 6990, "\u0120promise": 6991, "\u0120excess": 6992, "\u0120overw": 6993, "\u0120prime": 6994, "\u0120Houston": 6995, "erry": 6996, "\u0120Ms": 6997, "RS": 6998, "2012": 6999, "\u0120stores": 7000, "\u0120Olymp": 7001, "\u0120journey": 7002, "Although": 7003, "Sub": 7004, "\u0120Educ": 7005, "\u0120Chapter": 7006, "\u0120requests": 7007, "\u0120consumers": 7008, "\u0120tiny": 7009, "\u0120isol": 7010, "\u0120Fair": 7011, "ba": 7012, "\u0120YOU": 7013, "\u0120crash": 7014, "celer": 7015, "\u0120emotional": 7016, "\u0120goods": 7017, "\u0120elected": 7018, "\u0120moder": 7019, "\u0120Linux": 7020, "\u0120blocks": 7021, "\u0120island": 7022, "\u0120Society": 7023, "\u0120elections": 7024, "\u0120broadcast": 7025, "\u0120cheap": 7026, "\u0120nations": 7027, "\u0120seasons": 7028, "400": 7029, "\u0120waste": 7030, "\u0120Sat": 7031, "\u0120fields": 7032, "employ": 7033, "\u0120profile": 7034, "\u0120authors": 7035, "ALL": 7036, "\u0120Gra": 7037, "west": 7038, "\u0120Ty": 7039, "\u0120deaths": 7040, "\u0120vacc": 7041, "\u0120formed": 7042, "\u0120du": 7043, "\u0120ongoing": 7044, "\u0120Muslims": 7045, "elf": 7046, "igure": 7047, "\u0120assume": 7048, "\u0120Ukraine": 7049, "water": 7050, "\u0120coast": 7051, "\u0120voted": 7052, "gor": 7053, "\u0120AS": 7054, "\u0120Michigan": 7055, "aza": 7056, "\u0120Arm": 7057, "iro": 7058, "\u0120flex": 7059, "asters": 7060, "''": 7061, "\u0120welcome": 7062, "arl": 7063, "\u0120locations": 7064, "igation": 7065, "\u0120Fil": 7066, "\u0120buying": 7067, "\u0120architect": 7068, "\u0120harder": 7069, "\u0120Cub": 7070, "\u0120interface": 7071, "\u0120restaurant": 7072, "\u0120discover": 7073, "\u0120exceed": 7074, "\u0120favour": 7075, "gery": 7076, "\u0120duty": 7077, "\u0120pitch": 7078, "ador": 7079, "\u0120Mach": 7080, "boy": 7081, "\u0120responded": 7082, "\u0120extended": 7083, "hers": 7084, "Many": 7085, "raid": 7086, "ifer": 7087, "\u0120Ins": 7088, "Ser": 7089, "\u0120medium": 7090, "she": 7091, "\u0120Sports": 7092, "\u0120magazine": 7093, "utation": 7094, "\u0120limits": 7095, "\u0120Gall": 7096, "\u0120external": 7097, "razil": 7098, "\u0120younger": 7099, "tle": 7100, "\u0120remind": 7101, "\u0120CON": 7102, "\u0120immediate": 7103, "\u0120hidden": 7104, "\u0120volunte": 7105, "\u0120simpl": 7106, "odcast": 7107, "\u0120phase": 7108, "dr": 7109, "\u0120plot": 7110, "\u0120exposure": 7111, "RI": 7112, "ograp": 7113, "vin": 7114, "anish": 7115, "\u0120Acad": 7116, "\u0120Engine": 7117, "\u0120expansion": 7118, "\u0120Pay": 7119, "Your": 7120, "\u0120pushed": 7121, "\u0120Ell": 7122, "\u0120Head": 7123, "\u0120marketing": 7124, "\u0120AC": 7125, "ket": 7126, "\u0120hits": 7127, "\u0120gro": 7128, "\u0120Age": 7129, "\u0120Scot": 7130, "][": 7131, "\u0120stim": 7132, "\u0120iPhone": 7133, "\u012a\u0134": 7134, "\u0120narrow": 7135, "\u0120Getty": 7136, "\u0120Turkey": 7137, "\u0120perfectly": 7138, "\u0120enable": 7139, "utch": 7140, "\u0120precise": 7141, "\u0120regime": 7142, "\u0120shif": 7143, "\u0120compens": 7144, "gun": 7145, "div": 7146, "\u0120chosen": 7147, "\u0120Ken": 7148, "Any": 7149, "\u0120trees": 7150, "\u0120recommended": 7151, "\u0120Ren": 7152, "uable": 7153, "\u0120HT": 7154, "Follow": 7155, "EG": 7156, "\u0120Hand": 7157, "\u0120Kenn": 7158, "\u0120arguments": 7159, "\u0120exists": 7160, "\u0120bike": 7161, "\u0120Conserv": 7162, "\u0120breaking": 7163, "\u0120Gar": 7164, "\u0120crazy": 7165, "\u0120virtual": 7166, "aylor": 7167, "ixel": 7168, "\u01201980": 7169, "\u0120permission": 7170, "\u0120Series": 7171, "\u0120consumer": 7172, "\u0120closely": 7173, "called": 7174, "\u012054": 7175, "\u0120hopes": 7176, "\u0120array": 7177, "\u0120Win": 7178, "\u0120Labour": 7179, "\u0120spons": 7180, "\u0120Ire": 7181, "\u0120pow": 7182, "\u0120readers": 7183, "\u0120employment": 7184, "\u0120creature": 7185, "\u0120resulting": 7186, "\u0120accurate": 7187, "\u0120moments": 7188, "\u0120argued": 7189, "\u0120ped": 7190, "During": 7191, "\u012053": 7192, "\u0120Tal": 7193, "\u0120sought": 7194, "\u0120suffering": 7195, "\u0120icon": 7196, "lee": 7197, "\u0120($": 7198, "alian": 7199, "\u00c2\u00b0": 7200, "\u0120pra": 7201, "\u0120bonus": 7202, "(\"": 7203, "ko": 7204, "\u0120acting": 7205, "DE": 7206, "fall": 7207, "\u0120comparison": 7208, "\u0120smooth": 7209, "\u0120NAS": 7210, "upp": 7211, "\u0120Joseph": 7212, "eping": 7213, "\u0120Take": 7214, "\u0120Mid": 7215, "\u0120sending": 7216, "fast": 7217, "\u0120Fall": 7218, "\u0120dealing": 7219, "user": 7220, "\u0120Organ": 7221, "Co": 7222, "\u0120attached": 7223, "\u0120sees": 7224, "%.": 7225, "\u0120typical": 7226, "ART": 7227, "\u0120finds": 7228, "\u0120Asia": 7229, "umin": 7230, "\u0120Core": 7231, "\u0120Ent": 7232, "inent": 7233, "uce": 7234, "\u0120Blood": 7235, "\u0120Never": 7236, "\u0120emails": 7237, "\u0120highlight": 7238, "\u0120confront": 7239, "atus": 7240, "uted": 7241, "\u0120unus": 7242, "\u0120topic": 7243, "\u0120Adam": 7244, "\u0120ble": 7245, "ati": 7246, "\u0120understood": 7247, "Set": 7248, "struct": 7249, "TP": 7250, "\u0120mob": 7251, "aa": 7252, "\u0120Start": 7253, "pected": 7254, "sell": 7255, "\u0120dedicated": 7256, "\u0120CA": 7257, "uan": 7258, "\u0120songs": 7259, "escription": 7260, "\u0120tech": 7261, "\u0120rape": 7262, "\u0120aside": 7263, "\u0120grant": 7264, "\u012056": 7265, "sub": 7266, "\u0120argue": 7267, "\u0120containing": 7268, "\u0120schedule": 7269, "\u0120liberal": 7270, "\u0120publicly": 7271, "\u0120heavily": 7272, "\u0120Ut": 7273, "iner": 7274, "\u0120Section": 7275, "\u0120Care": 7276, "weet": 7277, "ls": 7278, "Dis": 7279, "\u00e2\u0136\u0122": 7280, "\u0120Follow": 7281, "Back": 7282, "\u0120IT": 7283, "\u0120bes": 7284, "ji": 7285, "\u0120Hit": 7286, "ested": 7287, "\u0120everybody": 7288, "\u0120Swed": 7289, "\u0120femin": 7290, "\u0120facilities": 7291, "\u0120conven": 7292, "Comp": 7293, "\u0120OS": 7294, "core": 7295, "\u0120anx": 7296, "\u0120division": 7297, "\u0120Cam": 7298, "\u0120Stan": 7299, "mates": 7300, "\u0120explore": 7301, "plom": 7302, "\u0120shares": 7303, "pload": 7304, "anes": 7305, "\u0120ideal": 7306, "eters": 7307, "\u0120Base": 7308, "\u0120plastic": 7309, "\u0120distinct": 7310, "\u0120Network": 7311, "\u0120Seattle": 7312, "\u0120trading": 7313, "ensus": 7314, "intend": 7315, "\u0120exhib": 7316, "\u0120initially": 7317, "\u0120Food": 7318, "\u0120thousand": 7319, "\u0120Business": 7320, "acter": 7321, "\u0120paragraph": 7322, "\u0120roughly": 7323, "\u0120www": 7324, "\u0120creative": 7325, "\u0120Conf": 7326, "\u0120consumption": 7327, "\u0120films": 7328, "agan": 7329, "\u0120obtain": 7330, "\u0120tall": 7331, "\u0120tor": 7332, "\u0120acknowled": 7333, "\u0120grown": 7334, "alo": 7335, "KE": 7336, "\u0120400": 7337, "enders": 7338, "taining": 7339, "UG": 7340, "\u0120suicide": 7341, "\u0120watched": 7342, "\u0120List": 7343, "ali": 7344, "rehens": 7345, "\u0120surrounding": 7346, "\u0120pip": 7347, "\u0120flying": 7348, "\u0120Java": 7349, "ordan": 7350, "\u0120serving": 7351, "inations": 7352, "post": 7353, "\u0120sho": 7354, "Av": 7355, "\u0120jail": 7356, "zy": 7357, "\u01201999": 7358, "\u0120>": 9609, "orous": 9610, "\u0120firms": 9611, "screen": 9612, "una": 9613, "\u0120embarrass": 9614, "ulse": 9615, "\u0120letting": 9616, "\u0120threw": 9617, "iley": 9618, "\u0120channels": 9619, "lan": 9620, "\u0120Vegas": 9621, "\u0120sear": 9622, "\u0120fantastic": 9623, "arre": 9624, "uzzle": 9625, "\u0120Der": 9626, "Those": 9627, "\u0120swing": 9628, "\u0120sheet": 9629, "index": 9630, "cover": 9631, "ogan": 9632, "\u0120variables": 9633, "\u0120Tech": 9634, "\u0120spoken": 9635, "achel": 9636, "\u0120Da": 9637, "\u0120Mountain": 9638, "\u0120loaded": 9639, "\u0120footage": 9640, "version": 9641, "\u0120unl": 9642, "\u0120Phoenix": 9643, "\u0120throwing": 9644, "\u0120firing": 9645, "\u0120tracking": 9646, "\u0120width": 9647, "\u0120struggling": 9648, "rooms": 9649, "otion": 9650, "\u0120monthly": 9651, "\u0120Server": 9652, "\u0120eggs": 9653, "open": 9654, "MC": 9655, "\u01201993": 9656, "\u0120hired": 9657, "\u0120stayed": 9658, "\u0120Allen": 9659, "\u0120stro": 9660, "\u012098": 9661, "step": 9662, "\u0120Turkish": 9663, "\u0120fabric": 9664, "isting": 9665, "\u0120Dom": 9666, "\u0120dates": 9667, "\u0120pron": 9668, "\u0120basketball": 9669, "\u0120lucky": 9670, "\u0120Arabia": 9671, "\u0120assumed": 9672, "esty": 9673, "\u0120affairs": 9674, "\u0120glad": 9675, "\u0120Indeed": 9676, "\u0120FA": 9677, "\u0120Word": 9678, "\u0120joining": 9679, "ifice": 9680, "pread": 9681, "irts": 9682, "\u0120Select": 9683, "\u0120populations": 9684, "aware": 9685, "\u0120nose": 9686, "\u0120complaints": 9687, "start": 9688, "\u0120scoring": 9689, "Thanks": 9690, "\u0120mining": 9691, "\u0120visitors": 9692, "SH": 9693, "\u0120damaged": 9694, "\u0120characteristics": 9695, "\u0120Pent": 9696, "DC": 9697, "\u012083": 9698, "\u0120Six": 9699, "rates": 9700, "\u0120flags": 9701, "\u0120Brew": 9702, "dog": 9703, "Mark": 9704, "////": 9705, "\u0120execution": 9706, "\u0120joke": 9707, "phones": 9708, "\u0120testimony": 9709, "\u0120obst": 9710, "QL": 9711, "\u0120Cut": 9712, "\u0120studied": 9713, "\u0120Nintendo": 9714, "icket": 9715, "\u0120NBC": 9716, "\u0120lad": 9717, "\u0120Bra": 9718, "\u0120Moh": 9719, "\u0120kernel": 9720, "\u0120overwhelming": 9721, "\u0120aged": 9722, "\u0120applicable": 9723, "\u0120Cond": 9724, "\u0120roads": 9725, "\u0120Block": 9726, "made": 9727, "odge": 9728, "\u0120commands": 9729, "\u0120offices": 9730, "veland": 9731, "\u0120tut": 9732, "\u0120receiver": 9733, "\u0120Fro": 9734, "\u0120shopping": 9735, "\u0120iP": 9736, "\u0120Stre": 9737, "\u0120ABC": 9738, "\u0120entertainment": 9739, "\u0120Bow": 9740, "orted": 9741, "Mc": 9742, "\u0120reads": 9743, "grad": 9744, "\u0120Collect": 9745, "\u0120\u00e2\u012a\u0134": 9746, "\u0120Capital": 9747, "ederation": 9748, "\u0120employer": 9749, "\u0120involvement": 9750, "\u0120anxiety": 9751, "alia": 9752, "\u0120roof": 9753, "\u0120Among": 9754, "\u0120Democrat": 9755, "\u0120stats": 9756, "\u0120Vill": 9757, "\u0120constitutional": 9758, "\u0120referring": 9759, "itty": 9760, "\u0120tackle": 9761, "outube": 9762, "\u0120backed": 9763, "\u0120Hong": 9764, "\u0120Broad": 9765, "\u0120ele": 9766, "\u0120Ott": 9767, "\u01201992": 9768, "hour": 9769, "achusetts": 9770, "Cal": 9771, "\u0120defeated": 9772, "\u012081": 9773, "esp": 9774, "\u0120seemingly": 9775, "was": 9776, "\u0120Jenn": 9777, "\u0120Kurd": 9778, "\u0120gene": 9779, "\u0120discount": 9780, "Ret": 9781, "ECT": 9782, "();": 9783, "\u0120clubs": 9784, "\u0120sid": 9785, "\u0120Marsh": 9786, "Check": 9787, "\u0120pp": 9788, "\u0120Eag": 9789, "idespread": 9790, "\u0120beings": 9791, "FT": 9792, "\u0120introduction": 9793, "\u0120Change": 9794, "ARD": 9795, "\u0120110": 9796, "adows": 9797, "ierce": 9798, "\u0120meal": 9799, "author": 9800, "\u0120Bang": 9801, "lahoma": 9802, "\u0120ranks": 9803, "2011": 9804, "????": 9805, "max": 9806, "\u0120collapse": 9807, "\u0120opens": 9808, "\u0120echo": 9809, "\u0120soph": 9810, "\u0120racist": 9811, "\u0120enormous": 9812, "\u0120waves": 9813, "\u0120tap": 9814, "\u0120comprehensive": 9815, ".--": 9816, "\u0120Roy": 9817, "\u0120farmers": 9818, "Related": 9819, "aired": 9820, "rones": 9821, "\u0120Crim": 9822, "\u0120proportion": 9823, "\u0120designs": 9824, "\u0120negotiations": 9825, "\u0120virtually": 9826, "\u0120Batman": 9827, "\u0120warn": 9828, "\u0120legitimate": 9829, "mate": 9830, "\u0120convention": 9831, ",,": 9832, "netic": 9833, "\u0120SD": 9834, "\u0120consistently": 9835, "\u0120compensation": 9836, "\u0120punishment": 9837, "\u0120ye": 9838, "\u0120tie": 9839, "\u0120Bureau": 9840, "irlf": 9841, "\u0120Bu": 9842, "\u0120Aren": 9843, "\u0120Philipp": 9844, "\u0120knife": 9845, "\u0120memories": 9846, "\u0120Ross": 9847, "\u0120angle": 9848, "\u012086": 9849, "\u0120Thunder": 9850, "\u0120rend": 9851, "\u0120Tour": 9852, "\u0120counts": 9853, "sung": 9854, "\u0120Imp": 9855, "\u0120educational": 9856, "\u0120accessible": 9857, "COM": 9858, "\u0120drew": 9859, "yer": 9860, "Gl": 9861, "amine": 9862, "ORT": 9863, "OB": 9864, "IB": 9865, "master": 9866, "\u0120trials": 9867, "ogy": 9868, "har": 9869, "\u0120Trust": 9870, "\u0120preferred": 9871, "irlfriend": 9872, "\u0120Nev": 9873, "\u0120bin": 9874, "\u0120cow": 9875, "Page": 9876, "\u0120signature": 9877, "\u0120BL": 9878, "700": 9879, "\u0120retired": 9880, "\u0120bytes": 9881, "\u0120neighb": 9882, "\u0120Legend": 9883, "\u0120devast": 9884, "\u0120suspected": 9885, "isons": 9886, "\u0120Pok\u00c3\u00a9mon": 9887, "scale": 9888, "\u0120capabilities": 9889, "\u0120revel": 9890, "\u0120cheese": 9891, "dy": 9892, "igrant": 9893, "\u0120failing": 9894, "bits": 9895, "\u0120Heroes": 9896, "\u0120Ghost": 9897, "\u0120Scient": 9898, "\u0120appointed": 9899, "uri": 9900, "\u0120institution": 9901, "\u0120expanded": 9902, "greg": 9903, "\u0120monitoring": 9904, "\u0120podcast": 9905, "\u0120coalition": 9906, "\u012096": 9907, "Jo": 9908, "\u0120stolen": 9909, "\u0120Sab": 9910, "\u0120stops": 9911, "\u0120holiday": 9912, "\u0120intr": 9913, "Car": 9914, "Black": 9915, "\u0120LGBT": 9916, "\u0120warming": 9917, "\u0120Anderson": 9918, "\u012089": 9919, "\u0120producer": 9920, "Med": 9921, "\u0120accuracy": 9922, "\u0120Marvel": 9923, "izabeth": 9924, "\u0120Patrick": 9925, "mony": 9926, "\u0120mini": 9927, "acles": 9928, "\u0120overt": 9929, "they": 9930, "\u0120membership": 9931, "\u0120Ven": 9932, "\u0120exch": 9933, "\u0120removal": 9934, "\u0120Dave": 9935, "TY": 9936, "mad": 9937, "\u0120Find": 9938, "\u0120adequ": 9939, "\u0120ec": 9940, "\u0120teeth": 9941, "\u0120emotion": 9942, "\u0120perm": 9943, "\u0120solely": 9944, "db": 9945, "\u0120extraord": 9946, "IGHT": 9947, "cal": 9948, "\u0120guidelines": 9949, "\u0120dying": 9950, "\u0120suspended": 9951, "\u0120Premier": 9952, "\u0120Anthony": 9953, "elve": 9954, "\u0120dad": 9955, "\u0120Eth": 9956, "\u0120Football": 9957, "\u0120abandoned": 9958, "\u0120<<": 9959, "\u0120march": 9960, "\u0120horror": 9961, "\u00e2\u0122\u00a6\"": 9962, "\u0120childhood": 9963, "\u0120campaigns": 9964, "\u0120lunch": 9965, "\u0120Albert": 9966, "block": 9967, "\u00e2\u0138\u012a\u00e2\u0138\u012a": 9968, "ounding": 9969, "\u0120bone": 9970, "organ": 9971, "aders": 9972, "\u0120Flash": 9973, "\u0120Drive": 9974, "\u0120tonight": 9975, "\u0120wars": 9976, "\u0120FL": 9977, "\u0120formation": 9978, "const": 9979, "News": 9980, "\u0120compe": 9981, "orious": 9982, "\u0120Staff": 9983, "\u0120discussions": 9984, "\u0120Protection": 9985, "\u0120Jam": 9986, "\u0120criteria": 9987, "\u0120installation": 9988, "\u0120accomplish": 9989, "izza": 9990, "\u0120publisher": 9991, "\u0120rescue": 9992, "\u0120Try": 9993, "ULL": 9994, "\u0120Som": 9995, "\u0120Hop": 9996, "oret": 9997, "ths": 9998, "ordon": 9999, "\u0120pocket": 10000, "\u0120Inv": 10001, "Download": 10002, "\u0120Crime": 10003, "\u0120bene": 10004, "\u0120Guide": 10005, "\u0120Assembly": 10006, "\u0120parameters": 10007, "IE": 10008, "\u0120Alexander": 10009, "\u0120concert": 10010, "\u0120Sche": 10011, "\u0120shoes": 10012, "\u0120visiting": 10013, "\u0120recall": 10014, "\u0120bub": 10015, "\u0120rural": 10016, "\u0120concrete": 10017, "\u0120Ros": 10018, "Next": 10019, "Russ": 10020, "\u0120loans": 10021, "\u0120Shield": 10022, "\u0120trem": 10023, "hemat": 10024, "kg": 10025, "\u0120Harris": 10026, "isition": 10027, "\u0120Move": 10028, "\u0120FC": 10029, "\u0120fate": 10030, "\u0120Cho": 10031, "\u0120tired": 10032, "\u0120principal": 10033, "hist": 10034, "iences": 10035, "athy": 10036, "\u0120sevent": 10037, "\u0120mood": 10038, "\u0120strategic": 10039, "\u0120diseases": 10040, "\u0120forum": 10041, "\u0120tempor": 10042, "\u0120headquarters": 10043, "Par": 10044, "ige": 10045, "flix": 10046, "\u0120guitar": 10047, "\u012094": 10048, "Only": 10049, "\u0120releases": 10050, "roph": 10051, "================================": 10052, "\u0120600": 10053, "\u0120Continue": 10054, "igate": 10055, "\u0120Crit": 10056, "system": 10057, "\u0120disabled": 10058, "\u0120unexpected": 10059, "ithub": 10060, "\u0120unclear": 10061, "\u0120Est": 10062, "\u0120contrad": 10063, "\u0120strategies": 10064, "ventures": 10065, "\u0120passage": 10066, "AME": 10067, "\u0120improving": 10068, "\u0120reveals": 10069, "\u0120decrease": 10070, "ova": 10071, "\u0120annoy": 10072, "\u0120Short": 10073, "\u0120Library": 10074, "\u0120cyber": 10075, "nell": 10076, "\u0120Hur": 10077, "\u0120CB": 10078, "\u0120photograp": 10079, "UI": 10080, "\u0120sed": 10081, "Ge": 10082, "\u012087": 10083, "\u0120diverse": 10084, "\u0120encouraged": 10085, "\u0120conspiracy": 10086, "\u0120birds": 10087, "\u0120operator": 10088, "\u0120handful": 10089, "\u0120classified": 10090, "?)": 10091, "\u0120dramatic": 10092, "\u0120investigators": 10093, "ito": 10094, "\u0120widespread": 10095, "\u0120Room": 10096, "----------------------------------------------------------------": 10097, "\u0120collective": 10098, "\u0120journalist": 10099, "String": 10100, "\u0120temperatures": 10101, "ila": 10102, "\u0120guid": 10103, "\u0120inspect": 10104, "\u0120missile": 10105, "\u0120Mayor": 10106, "\u0120manual": 10107, "\u0120simultane": 10108, "\u0120ratings": 10109, "\u0120suck": 10110, "\u012097": 10111, "\u0120universal": 10112, "\u0120pharm": 10113, "\u0120disrupt": 10114, "iano": 10115, "AV": 10116, "\u0120ft": 10117, "\u0120statist": 10118, "olds": 10119, "\u0120Walker": 10120, "php": 10121, "\u0120undert": 10122, "\u0120Las": 10123, "ishop": 10124, "ntil": 10125, "reshold": 10126, "\u0120Whether": 10127, "Ms": 10128, "\u0120deny": 10129, "\u0120Cloud": 10130, "\u0120provider": 10131, "\u0120surviv": 10132, "\u0120Update": 10133, "has": 10134, "\u0120mistakes": 10135, "charge": 10136, "pled": 10137, "rity": 10138, "\u0120node": 10139, "\u0120Massachusetts": 10140, "ools": 10141, "lication": 10142, "\u0120fails": 10143, "emale": 10144, "ori": 10145, "backs": 10146, "\u0120shirt": 10147, "\u0120''": 10148, "\u0120NAT": 10149, "\u0120waters": 10150, "elson": 10151, "\u0120ease": 10152, "\u0120scar": 10153, "\u0120contents": 10154, "mind": 10155, "\u0120contribution": 10156, "\u0120shr": 10157, "\u0120handed": 10158, "\u0120stability": 10159, "\u0120trave": 10160, "Em": 10161, "\u0120mirror": 10162, "123": 10163, "\u0120weigh": 10164, "\u0120fiction": 10165, "ouver": 10166, "istant": 10167, "rition": 10168, "\u0120Fed": 10169, "\u0120physically": 10170, "\u0120stake": 10171, "\u0120Article": 10172, "\u0120Arc": 10173, "\u0120Lewis": 10174, "\u0120Mind": 10175, "\u0120demonstrate": 10176, "\u0120profits": 10177, "vision": 10178, "omic": 10179, "olid": 10180, "\u0120battles": 10181, "\u0120drives": 10182, "\u0120eastern": 10183, "\u0120Sony": 10184, "!!!": 10185, "aration": 10186, "vard": 10187, "\u0120GL": 10188, "portation": 10189, "\u012092": 10190, "\u0120lawmakers": 10191, "\u0120protecting": 10192, "\u0120EPA": 10193, "\u0120yeah": 10194, "\u0120shame": 10195, "olph": 10196, "even": 10197, "xit": 10198, "\u0120attach": 10199, "\u0120representing": 10200, "\u0120obs": 10201, "\u0120Utah": 10202, "iffs": 10203, "\u0120Freedom": 10204, "\u00c3\u00b3": 10205, "AK": 10206, "\u0120incidents": 10207, "itage": 10208, "\u0120viewers": 10209, "cd": 10210, "\u0120mouse": 10211, "\u0120clar": 10212, "\u0120accordance": 10213, "\u0120bot": 10214, "cor": 10215, "\u0120Summer": 10216, "held": 10217, "\u0120innocent": 10218, "\u0120initiative": 10219, "ols": 10220, "________________________________": 10221, "\u0120spots": 10222, "pace": 10223, "\u0120conventional": 10224, "\u0120corporations": 10225, "\u0120blocked": 10226, "HD": 10227, "attered": 10228, "\u0120refers": 10229, "\u0120buck": 10230, "\u0120Digital": 10231, "120": 10232, "\u0120topics": 10233, "TF": 10234, "\u00c4\u0123": 10235, "brid": 10236, "reement": 10237, "\u0120underlying": 10238, "\u0120Member": 10239, "\u0120investigating": 10240, "\u0120pregnancy": 10241, "\u0120touchdown": 10242, "\u0120Band": 10243, "\u0120Caller": 10244, "\u0120instances": 10245, "PP": 10246, "wa": 10247, "Good": 10248, "\u01201991": 10249, "\u0120Cold": 10250, "\u0120fears": 10251, "\u0120remarks": 10252, "\u0128\u0134": 10253, "atal": 10254, "\u0120mit": 10255, "\u0120experiments": 10256, "ipt": 10257, "Color": 10258, "indu": 10259, "Update": 10260, "\u012093": 10261, "Ag": 10262, "\u0120\u00e5": 10263, "ancouver": 10264, "Both": 10265, "\u0120judges": 10266, "Object": 10267, "\u0120stere": 10268, "umbn": 10269, "\u0120participation": 10270, "\u0120Stars": 10271, "\u0120Jere": 10272, "\u0120weekly": 10273, "\u0120Ban": 10274, "\u0120conversations": 10275, "\u0120Pitt": 10276, "uz": 10277, "\u0120Indiana": 10278, "\u0120Kick": 10279, "\u0120infection": 10280, "\u0120heroes": 10281, "\u0120settled": 10282, "\u0120strip": 10283, "\u0120hal": 10284, "\u0120dump": 10285, "\u0120Sci": 10286, "\u0120les": 10287, "\u0120references": 10288, "\u0120URL": 10289, "\u0120Bridge": 10290, "\u0120wanting": 10291, "Force": 10292, "\u0120exclus": 10293, "Meanwhile": 10294, "mn": 10295, "\u0120gentle": 10296, "maker": 10297, "senal": 10298, "\u0120Gro": 10299, "ouri": 10300, "\u0120Rain": 10301, "\u0120Alliance": 10302, "\u0120lift": 10303, "ela": 10304, "SD": 10305, "\u0120Cleveland": 10306, "\u0120ranked": 10307, "\u0120stadium": 10308, "\u0120deadly": 10309, "\u00e4\u00b8": 10310, "\u0120riding": 10311, "aria": 10312, "\u0120Armor": 10313, "\u0120documentation": 10314, "\u0120Greece": 10315, "reek": 10316, "\u0120lens": 10317, "\u0120Sa": 10318, "\u0120gross": 10319, "\u0120Emer": 10320, "agers": 10321, "\u0120Dub": 10322, "\u0120Rh": 10323, "\u0120AMD": 10324, "\u0120arrival": 10325, "\u0120desert": 10326, "\u0120supplement": 10327, "\u0120Resp": 10328, "\u0120knee": 10329, "\u0120margin": 10330, "font": 10331, "ogg": 10332, "2010": 10333, "\u0120Pir": 10334, "\u0120Prom": 10335, "ivals": 10336, "\u0120intake": 10337, "\u0120differently": 10338, "ugs": 10339, "\u0120bits": 10340, "cluded": 10341, "\u0120searching": 10342, "\u0120Du": 10343, "umble": 10344, "\u0120functional": 10345, "\u0120Baltimore": 10346, "\u0120Could": 10347, "\u0120desired": 10348, "\u0120circuit": 10349, "\u0120Lyn": 10350, "\u0120GO": 10351, "\u0120False": 10352, "repre": 10353, "':": 10354, "alties": 10355, "\u0120minim": 10356, "\u0120drove": 10357, "\u0120Should": 10358, "\u0120hip": 10359, "\u0120pros": 10360, "\u0120utility": 10361, "\u0120Nature": 10362, "\u0120Mode": 10363, "President": 10364, "opp": 10365, "rat": 10366, "formance": 10367, "\u0120concentration": 10368, "\u0120font": 10369, "\u0120Bud": 10370, "\u0120amid": 10371, "\u0120revers": 10372, "\u0120ML": 10373, "Bar": 10374, "\u0120interaction": 10375, "\u0120jurisd": 10376, "\u0120spells": 10377, "dep": 10378, "fil": 10379, "\u0120civilians": 10380, "utter": 10381, "\u0120Cooper": 10382, "\u0120Below": 10383, "\u0120entrance": 10384, "\u0120convert": 10385, "\u0120controversy": 10386, "owered": 10387, "\u0120contrary": 10388, "\u0120arc": 10389, "\u0120Executive": 10390, "\u0120Officer": 10391, "\u0120packages": 10392, "\u0120progressive": 10393, "width": 10394, "\u0120reserved": 10395, "vol": 10396, "\u0120Samsung": 10397, "\u0120printed": 10398, "\u0120centers": 10399, "\u0120introduce": 10400, "\u0120Kennedy": 10401, "\u0120odds": 10402, "\u0120surely": 10403, "\u0120independence": 10404, "\u0120passengers": 10405, "reprene": 10406, "\u0120Beh": 10407, "\u0120loves": 10408, "\u0120ESPN": 10409, "\u0120facilit": 10410, "\u0120identical": 10411, "\u0120doct": 10412, "\u0120partnership": 10413, "conf": 10414, "\u0120Hide": 10415, "\u0120confused": 10416, "\u0120Cow": 10417, "Men": 10418, "\u0120wrest": 10419, "\u0120Iraqi": 10420, "\u0120holes": 10421, "\u0120Studies": 10422, "\u0120pregnant": 10423, "hard": 10424, "\u0120signals": 10425, "IX": 10426, "\u0120pulling": 10427, "\u0120graduate": 10428, "\u0120nominee": 10429, "Date": 10430, "\u0120permitted": 10431, "\u0120\u00e2\u0124\u00ac": 10432, "\u0120Oklahoma": 10433, "Start": 10434, "\u0120authorized": 10435, "\u0120alarm": 10436, "\u0120Cos": 10437, "van": 10438, "\u0120generations": 10439, "cular": 10440, "\u0120dragon": 10441, "\u0120Software": 10442, "\u0120Edward": 10443, "\u0120controller": 10444, "Sen": 10445, "gered": 10446, "\u0120Vik": 10447, "\u0120approached": 10448, "Thank": 10449, "\u0120cance": 10450, "\u0120formula": 10451, "\u0120Small": 10452, "\u0120weakness": 10453, "\u0120ramp": 10454, "itudes": 10455, "jud": 10456, "\u0120brilliant": 10457, "\u0120accus": 10458, "source": 10459, "\u0120800": 10460, "\u0120Evil": 10461, "Sw": 10462, "\u0120homeless": 10463, "week": 10464, "iens": 10465, "rics": 10466, "\u0120Third": 10467, "TO": 10468, "\u0120organic": 10469, "\u0120presentation": 10470, "agh": 10471, "\u0120Download": 10472, "vation": 10473, "\u0120assembly": 10474, "orable": 10475, "holders": 10476, "\u0120Bernie": 10477, "\u0120Help": 10478, "\u0120tong": 10479, "\u0120Fight": 10480, "\u0120beach": 10481, "Book": 10482, "\u0120Lic": 10483, "\u0120rush": 10484, "\u0120Round": 10485, "oup": 10486, "\u0120Marx": 10487, "\u0120calculated": 10488, "\u0120Devil": 10489, "\u0120Sarah": 10490, "\u0120occasionally": 10491, "\u0120bullet": 10492, "Available": 10493, "gate": 10494, "\u012091": 10495, "\u0120hosp": 10496, "\u0120promises": 10497, "\u0120HIV": 10498, "\u0120Stadium": 10499, "\u0120Stock": 10500, "\u0120Corporation": 10501, "gage": 10502, "NG": 10503, "\u0120Credit": 10504, "\u0120sne": 10505, "ibl": 10506, "\u0120accum": 10507, "such": 10508, "\u0120terrorists": 10509, "\u0120consciousness": 10510, "\u0120Zh": 10511, "\u0120drama": 10512, "oola": 10513, "piration": 10514, "\u0120labour": 10515, "\u0120Nin": 10516, "\u0120utter": 10517, "\u0120democratic": 10518, "\u0120assass": 10519, "ilation": 10520, "\u0120gest": 10521, "\u0120abroad": 10522, "\u0120metab": 10523, "\u0120sorts": 10524, "\u0120flav": 10525, "UB": 10526, "\u0120mg": 10527, "\u0120Nothing": 10528, "\u0120Od": 10529, "\u0120musical": 10530, "2009": 10531, "\u0120drops": 10532, "ocated": 10533, "ateral": 10534, "000000": 10535, "\u0120gre": 10536, "\u0120equality": 10537, "\u0120burden": 10538, "\u0120vig": 10539, "\u0120Leader": 10540, "------------": 10541, "\u0120ceremony": 10542, "\u0120fighter": 10543, "\u0120actors": 10544, "\u0120\u00e6": 10545, "aman": 10546, "Fi": 10547, "\u0120align": 10548, "puter": 10549, "\u0120elder": 10550, "\u0120NSA": 10551, "\u0120representation": 10552, "\u0120Ontario": 10553, "ITH": 10554, "usalem": 10555, "\u0120harassment": 10556, "itzer": 10557, "\u0120symp": 10558, "\u0120boxes": 10559, "\u0120DR": 10560, "\u0120manifest": 10561, "atre": 10562, "\u0120^": 10563, "\u0120dies": 10564, "leton": 10565, "\u0120missions": 10566, "ethe": 10567, "\u0120resolve": 10568, "\u0120followers": 10569, "\u0120asc": 10570, "\u0120km": 10571, "lord": 10572, "ammed": 10573, "\u0120silent": 10574, "\u0120Associated": 10575, "\u0120timing": 10576, "\u0120prisoners": 10577, "\u0120Kings": 10578, "\u0120Five": 10579, "\u0120tower": 10580, "\u0120approaches": 10581, "\u0120precisely": 10582, "\u0120bureau": 10583, "\u0120Mother": 10584, "\u0120Iss": 10585, "\u0120keyboard": 10586, "itual": 10587, "\u0120funded": 10588, "\u0120staying": 10589, "\u0120psychological": 10590, "\u0120mile": 10591, "\u0120Leon": 10592, "\u0120Barb": 10593, "will": 10594, "\u0120wider": 10595, "\u0120Atlantic": 10596, "\u0120till": 10597, "\u0120Rome": 10598, "rot": 10599, "\u0120accompan": 10600, "\u0120flour": 10601, "aco": 10602, "World": 10603, "\u0120Express": 10604, "\u0120Yu": 10605, "Cor": 10606, "\u0120pleased": 10607, "party": 10608, "\u0120pointing": 10609, "\u0120inflation": 10610, "\u0120roy": 10611, "\u0120),": 10612, "ainer": 10613, "\u0120wedding": 10614, "ormon": 10615, "\u0120requiring": 10616, "\u0120qualified": 10617, "\u0120segment": 10618, "END": 10619, "\u0120sizes": 10620, "eals": 10621, "\u0120corrupt": 10622, "assador": 10623, "\u0120celeb": 10624, "\u0120dreams": 10625, "\u0120Mess": 10626, "\u0120checking": 10627, "\u0120Version": 10628, "\u0120preparing": 10629, "\u0120actively": 10630, "\u0120Diff": 10631, "\u0120lux": 10632, "\u0120Winter": 10633, "acteria": 10634, "\u0120NE": 10635, "\u0120deputy": 10636, "\u0120transgender": 10637, "\u0120summary": 10638, "\u0120inher": 10639, "eries": 10640, "char": 10641, "\u0120Yan": 10642, "\u0120knock": 10643, "\u0120Path": 10644, "\u0120lip": 10645, "roller": 10646, "\u0120impression": 10647, "\u0120celebrate": 10648, "\u0120slide": 10649, "\u0120guests": 10650, "\u0120clip": 10651, "FS": 10652, "\u0120savings": 10653, "\u0120captain": 10654, "\u0120legacy": 10655, "\u0120Denver": 10656, "\u0120wounded": 10657, "taboola": 10658, "ACT": 10659, "\u0120pursue": 10660, "\u0120oxy": 10661, "\u0120q": 10662, "\u0120semi": 10663, "\u0120Need": 10664, "\u0120Affairs": 10665, "\u0120obsc": 10666, "\u0120checked": 10667, "\u0120dual": 10668, "Code": 10669, "\u0120MD": 10670, "lem": 10671, "ulty": 10672, "\u0120\u00c2\u00a9": 10673, "\u0120Elizabeth": 10674, "\u0120centuries": 10675, "arded": 10676, "src": 10677, "\u0120evident": 10678, "ennis": 10679, "atin": 10680, "\u0120unemployment": 10681, "\u0120Mario": 10682, "\u0120intim": 10683, "Christ": 10684, "\u0120biological": 10685, "\u0120soldier": 10686, "\u0120Added": 10687, "\u0120math": 10688, "\u0120Gil": 10689, "\u0120bias": 10690, "\u0120dating": 10691, "\u0120Ocean": 10692, "\u0120mice": 10693, "Mus": 10694, "hire": 10695, "\u0120Tes": 10696, "Server": 10697, "limited": 10698, "Size": 10699, "\u0120meters": 10700, "\u0120rocket": 10701, "essee": 10702, "\u0120certificate": 10703, "\u0120Iranian": 10704, "ASS": 10705, "\u0120grid": 10706, "Dec": 10707, "\u0120rolling": 10708, "commun": 10709, "\u0120Sweden": 10710, "bury": 10711, "\u0120tissue": 10712, "\u0120racism": 10713, "\u0120Local": 10714, "\u0120mystery": 10715, "\u0120examine": 10716, "\u0120stem": 10717, "\u0120sits": 10718, "\u0120hoped": 10719, "oting": 10720, "\u0120dialogue": 10721, "\u0120persu": 10722, "Watch": 10723, "lay": 10724, "MAN": 10725, "\u0120chronic": 10726, "\u0120Portland": 10727, "market": 10728, "\u0120SEC": 10729, "\u0120parallel": 10730, "\u0120scandal": 10731, "\u0120carries": 10732, "\u0120phenomenon": 10733, "human": 10734, "acker": 10735, "\u0120Ox": 10736, "\u0120retirement": 10737, "tainment": 10738, "ovie": 10739, "\u0120Gear": 10740, "\u0120duties": 10741, "\u0120dose": 10742, "\u0120scroll": 10743, "MB": 10744, "inf": 10745, "\u0120sauce": 10746, "\u0120landscape": 10747, "reddit": 10748, "\u0120Championship": 10749, "\u0120Reddit": 10750, "alid": 10751, "\u0120coin": 10752, "\u0120overs": 10753, "\u0120posting": 10754, "about": 10755, "\u0120fel": 10756, "andy": 10757, "\u0120bold": 10758, "\u0120focusing": 10759, "effect": 10760, "GR": 10761, "\u0120deemed": 10762, "\u0120recommendations": 10763, "\u0120stepped": 10764, "\u0120voter": 10765, "\u0120Deep": 10766, "\u0120Instagram": 10767, "\u0120moderate": 10768, "\u0120Maryland": 10769, "\u0120restricted": 10770, "\u0120MB": 10771, "\u0120Chall": 10772, "\u0120tob": 10773, "\u0120cir": 10774, "\u0120Occ": 10775, "\u0120Ever": 10776, "\u0120collaps": 10777, "INFO": 10778, "=-": 10779, "\u0120Pict": 10780, "\u0120Account": 10781, "nc": 10782, "\u0120ought": 10783, "\u0120export": 10784, "\u0120drunk": 10785, "('": 10786, "\u0120wise": 10787, "\u0120Mort": 10788, "necess": 10789, "\u0120ancest": 10790, "\u0120Incre": 10791, "\u0120frequent": 10792, "mir": 10793, "\u0120interpretation": 10794, "\u0120dependent": 10795, "\u0120coins": 10796, "\u0120Bol": 10797, "Video": 10798, "\u0120Justin": 10799, "\u0120fatal": 10800, "\u0120cooking": 10801, "\u0120confusion": 10802, "ipher": 10803, "\u0120custody": 10804, "\u0120Morgan": 10805, "omach": 10806, "\u0120Governor": 10807, "\u0120restaurants": 10808, "eling": 10809, "\u0120acknowledged": 10810, "\u0120ther": 10811, "\u0120genes": 10812, "ching": 10813, "Hey": 10814, "\u0120tactics": 10815, "\u0120Mexican": 10816, "\u0120vend": 10817, "\u0120hes": 10818, "quer": 10819, "\u0120noting": 10820, "\u0120Cameron": 10821, "\u0120targeting": 10822, "rock": 10823, "\u0120credits": 10824, "\u0120emotions": 10825, "\u0120representatives": 10826, "news": 10827, "\u0120legislative": 10828, "\u0120removing": 10829, "\u0120tweeted": 10830, "\u0120Carter": 10831, "\u0120Fixed": 10832, "\u0120forcing": 10833, "\u0120speaker": 10834, "\u0120males": 10835, "\u0120Vietnam": 10836, "lined": 10837, "\u0120concepts": 10838, "\u0120voices": 10839, "oir": 10840, "\u0120Trib": 10841, "Whe": 10842, "\u0120Jerusalem": 10843, "\u0120Sant": 10844, "\u0120cul": 10845, "\u0120lady": 10846, "\u0120Hawai": 10847, "\u0120arts": 10848, "\u0120Inn": 10849, "\u0120Machine": 10850, "\u0120Emperor": 10851, "\u0120slot": 10852, "gly": 10853, "\u0120Process": 10854, "III": 10855, "\u0120athletes": 10856, "\u0120Temple": 10857, "\u0120Represent": 10858, "\u0120presc": 10859, "\u0120tons": 10860, "\u0120golden": 10861, "\u0120punch": 10862, "\u0120GR": 10863, "iverpool": 10864, "\u0120enact": 10865, "\u0120lobby": 10866, "\u0120mos": 10867, "\u0120picking": 10868, "\u0120lifetime": 10869, "\u0120cognitive": 10870, "Each": 10871, "zo": 10872, "\u0120dub": 10873, "\u0120consists": 10874, "oln": 10875, "\u0120festival": 10876, "amous": 10877, "\u0120intellig": 10878, "words": 10879, "\u0120Smart": 10880, "\u0120dele": 10881, "\u0120lapt": 10882, "\u0120magical": 10883, "\u0120Sin": 10884, "bus": 10885, "urities": 10886, "ighth": 10887, "\u0120Ruby": 10888, "\u0120Sure": 10889, "olving": 10890, "\u0120jun": 10891, "OST": 10892, "\u0120imposed": 10893, "\u0120astron": 10894, "\u0120correl": 10895, "\u0120NS": 10896, "\u0120Kit": 10897, "\u0120Future": 10898, "burn": 10899, "\u0120immune": 10900, "ocus": 10901, "\u0120courses": 10902, "\u0120String": 10903, "\u0120lean": 10904, "\u0120ghost": 10905, "\u0120outcomes": 10906, "\u0120expense": 10907, "\u0120everyday": 10908, "\u0120acceptable": 10909, "Ah": 10910, "\u0120equipped": 10911, "\u0120orange": 10912, "FR": 10913, "\u0120Dutch": 10914, "Though": 10915, "\u0120Rank": 10916, "QU": 10917, "\u0120Roberts": 10918, "what": 10919, "rend": 10920, "\u0120disappear": 10921, "\u0120spawn": 10922, "\u0120Lam": 10923, "ois": 10924, "\u0120deserve": 10925, "\u0120minimal": 10926, "\u0120nervous": 10927, "\u0120Would": 10928, "\u0120rook": 10929, "\u0120Vancouver": 10930, "\u0120resign": 10931, "shire": 10932, "\u0120Works": 10933, "\u0120Build": 10934, "\u0120affordable": 10935, "\u0120Gary": 10936, "\u0120Arena": 10937, "\u0120hanging": 10938, "\u0120implications": 10939, "\u0120Song": 10940, "\u0120maintaining": 10941, "\u0120guards": 10942, "CON": 10943, "\u0120derived": 10944, "\u0120executed": 10945, "\u0120theories": 10946, "\u0120quoted": 10947, "\u0120Andre": 10948, "oga": 10949, "seless": 10950, "info": 10951, "\u0120Belg": 10952, "\u0120tears": 10953, "\u0120Surv": 10954, "\u0120birthday": 10955, "igious": 10956, "immer": 10957, "\u0120spectrum": 10958, "\u0120architecture": 10959, "\u0120recruit": 10960, "arma": 10961, "Table": 10962, "\u0120monsters": 10963, "\u0120Gov": 10964, "\u0120destination": 10965, "\u0120attractive": 10966, "\u0120foss": 10967, "\u0120Moreover": 10968, "\u0120presents": 10969, "THE": 10970, "\u0120reply": 10971, "pton": 10972, "\u0120cum": 10973, "\u0120delight": 10974, "\u0120affects": 10975, "\u0120donations": 10976, "\u0120Toy": 10977, "\u0120Him": 10978, "MENT": 10979, "\u0120overcome": 10980, "itched": 10981, "\u0120Fantasy": 10982, "\u0120Hat": 10983, "\u0120Beast": 10984, "bott": 10985, "\u0120investigations": 10986, "Run": 10987, "\u0120hunting": 10988, "di": 10989, "fund": 10990, "\u0120sessions": 10991, "estyle": 10992, "\u0120portray": 10993, "oids": 10994, "Yeah": 10995, "\u0120communicate": 10996, "\u0120comedy": 10997, "\u0120Yang": 10998, "\u0120belt": 10999, "\u0120Marine": 11000, "\u0120predicted": 11001, "Play": 11002, "\u0120importantly": 11003, "\u0120remarkable": 11004, "\u0120eliminate": 11005, "David": 11006, "\u0120bind": 11007, "VID": 11008, "\u0120advocates": 11009, "\u0120Gaza": 11010, "imp": 11011, "DB": 11012, "\u0120Na": 11013, "\u0120Similar": 11014, "IES": 11015, "\u0120charity": 11016, "vas": 11017, "math": 11018, "\u0120\u00e2\u0138": 11019, "oker": 11020, "ndum": 11021, "\u0120caps": 11022, "\u0120Hal": 11023, "2000": 11024, "ean": 11025, "\u0120fleet": 11026, "\u0120recre": 11027, "Right": 11028, "\u0120sleeping": 11029, "ijing": 11030, "kind": 11031, "\u0120designated": 11032, "\u00c3\u00a4": 11033, "\u0120animation": 11034, "kee": 11035, "\u0120Introdu": 11036, "\u0120/>": 11037, "\u0120delayed": 11038, "\u0120tremend": 11039, "\u0120curious": 11040, "Use": 11041, "\u0120lect": 11042, "dam": 11043, "\u0120innovation": 11044, "\u0120Points": 11045, "\u0120loading": 11046, "\u0120dispute": 11047, "ctic": 11048, "irds": 11049, "\u0120BY": 11050, "\u0120nurs": 11051, "\u0120Value": 11052, "IONS": 11053, "\u0120Hum": 11054, "\u0120template": 11055, "mers": 11056, "\u0120appearances": 11057, "\u0120Entertainment": 11058, "\u0120translation": 11059, "\u0120sake": 11060, "\u0120beneath": 11061, "\u0120inhib": 11062, "\u0120euro": 11063, "abetes": 11064, "\u0120studying": 11065, "\u0120Mas": 11066, "\u0120perceived": 11067, "\u0120examined": 11068, "\u0120eager": 11069, "\u0120coaches": 11070, "\u0120imper": 11071, "chi": 11072, "\u0120produces": 11073, "\").": 11074, "\u0120Everyone": 11075, "\u0120municip": 11076, "\u0120girlfriend": 11077, "\u0120hire": 11078, "\u0120Vice": 11079, "\u0120suitable": 11080, "opy": 11081, "\u0120inequ": 11082, "\u0120Duke": 11083, "fish": 11084, "first": 11085, "\u0120Obs": 11086, "\u0120interior": 11087, "\u0120Bruce": 11088, "\u0120Ry": 11089, "\u0120analys": 11090, "\u0120considerable": 11091, "\u0120forecast": 11092, "\u0120fert": 11093, "orship": 11094, "\u0120Drug": 11095, "\u0120ALL": 11096, ":\"": 11097, "thur": 11098, "\u0120Mail": 11099, "\u0120ballot": 11100, "\u0120instantly": 11101, "\u0120Channel": 11102, "\u0120picks": 11103, "\u01201989": 11104, "\u0120tent": 11105, "oli": 11106, "\u0120civilian": 11107, "bling": 11108, "ello": 11109, "bu": 11110, "\u0120inch": 11111, "\u0120logo": 11112, "\u0120cooperation": 11113, "\u0120walks": 11114, "\u0120investments": 11115, "\u0120imprison": 11116, "\u0120Festival": 11117, "\u0120Ky": 11118, "\u0120legally": 11119, "\u0120gri": 11120, "charg": 11121, "Sl": 11122, "\u0120threatening": 11123, "duction": 11124, "flow": 11125, "\u0120dismissed": 11126, "ibraries": 11127, "cap": 11128, "ele": 11129, "\u0120McG": 11130, "\u0120Harvard": 11131, "\u0120Conservative": 11132, "\u0120CBS": 11133, "png": 11134, "\u0120roots": 11135, "\u0120Having": 11136, "umbled": 11137, "\u0120Fun": 11138, "\\/": 11139, "\u0120Search": 11140, "plex": 11141, "\u0120discussing": 11142, "\u0120continu": 11143, "\u0120Tai": 11144, "\u0120Wik": 11145, "Free": 11146, "fit": 11147, "\u0120refuse": 11148, "\u0120managing": 11149, "\u0120synd": 11150, "ipedia": 11151, "walk": 11152, "\u0120professionals": 11153, "\u0120guidance": 11154, "\u0120universities": 11155, "\u0120assemb": 11156, "untu": 11157, "Finally": 11158, "ASE": 11159, "\u0120Auto": 11160, "\u0120Had": 11161, "\u0120anniversary": 11162, "LD": 11163, "\u0120Dur": 11164, "\u0120Ultimate": 11165, "ihad": 11166, "product": 11167, "\u0120transit": 11168, "\u0120restore": 11169, "\u0120explaining": 11170, "\u0120asset": 11171, "\u0120transferred": 11172, "\u0120burst": 11173, "apolis": 11174, "\u0120Magazine": 11175, "\u0120Cra": 11176, "\u0120BR": 11177, "gged": 11178, "\u0120HE": 11179, "Mich": 11180, "bet": 11181, "\u0120Lady": 11182, "ylum": 11183, "erves": 11184, "\u0120meets": 11185, "white": 11186, "Log": 11187, "\u0120corresponding": 11188, "\u0120insisted": 11189, "GG": 11190, "\u0120surrounded": 11191, "\u0120tens": 11192, "\u0120lane": 11193, "\u0120coinc": 11194, "home": 11195, "\u0120existed": 11196, "ected": 11197, "\u0120Double": 11198, "lamm": 11199, "\u0120skept": 11200, "exp": 11201, "\u0120perception": 11202, "iev": 11203, "\u0120Being": 11204, "oft": 11205, "\u0120adopt": 11206, ".:": 11207, "];": 11208, "Windows": 11209, "\u0120satellite": 11210, "ASH": 11211, "\u0120infant": 11212, "description": 11213, "\u0120Meanwhile": 11214, "cm": 11215, "oca": 11216, "\u0120Treat": 11217, "actor": 11218, "\u0120tobacco": 11219, "\u0120Norm": 11220, "emption": 11221, "\u0120flesh": 11222, "\u0120je": 11223, "oop": 11224, "\u0120Heaven": 11225, "\u0120beating": 11226, "anim": 11227, "\u0120gathering": 11228, "\u0120cultiv": 11229, "GO": 11230, "abe": 11231, "\u0120Jonathan": 11232, "\u0120Safety": 11233, "\u0120badly": 11234, "prot": 11235, "\u0120choosing": 11236, "\u0120contacted": 11237, "\u0120quit": 11238, "\u0120distur": 11239, "\u0120stir": 11240, "\u0120token": 11241, "Det": 11242, "\u0120Pa": 11243, "\u0120functionality": 11244, "003": 11245, "some": 11246, "\u0120limitations": 11247, "\u0120meth": 11248, "build": 11249, "config": 11250, "NT": 11251, "rell": 11252, "blem": 11253, "\u0120Mom": 11254, "\u0120veterans": 11255, "\u0120Hu": 11256, "\u0120trends": 11257, "arer": 11258, "\u0120Given": 11259, "\u0120Caption": 11260, "may": 11261, "AST": 11262, "\u0120wondering": 11263, "\u0120Clark": 11264, "normal": 11265, "\u0120separated": 11266, "\u0120desp": 11267, "stic": 11268, "brew": 11269, "\u0120relating": 11270, "\u0120Nik": 11271, "\u0120Farm": 11272, "\u0120enthusi": 11273, "good": 11274, "deb": 11275, "\u0120activist": 11276, "\u0120mart": 11277, "\u0120explosion": 11278, "\u0120Economic": 11279, "Link": 11280, "\u0120insight": 11281, "\u0120convenient": 11282, "\u0120counterpart": 11283, "support": 11284, "\u0120Virt": 11285, "agen": 11286, "\u0120Tennessee": 11287, "\u0120Simon": 11288, "\u0120Award": 11289, "OCK": 11290, "\u0120Figure": 11291, "\u0120overseas": 11292, "\u0120pride": 11293, "\u0120Cas": 11294, "note": 11295, "mg": 11296, "Current": 11297, "\u0120displays": 11298, "content": 11299, "\u0120traveling": 11300, "\u0120hospitals": 11301, "\u0120Financial": 11302, "\u0120Past": 11303, "\u0120defendant": 11304, "\u0120streaming": 11305, "mble": 11306, "\u0120Berlin": 11307, "uki": 11308, "\u0120distribut": 11309, "\u0120antib": 11310, "\u0120chocolate": 11311, "\u0120Castle": 11312, "\u0120interrupt": 11313, "\u0120Row": 11314, "\u0120conversion": 11315, "\u0120bugs": 11316, "\u0120Rather": 11317, "liest": 11318, "LY": 11319, "\u0120Jean": 11320, "common": 11321, "akh": 11322, "\u0120130": 11323, "otton": 11324, "\u0120Dean": 11325, "\u0120amendment": 11326, "\u0120gameplay": 11327, "\u0120Warren": 11328, "oda": 11329, "\u0120highlights": 11330, "\u0120irre": 11331, "\u0120NATO": 11332, "\u0120balls": 11333, "\u0120demanding": 11334, "URE": 11335, "\u0120Luke": 11336, "Figure": 11337, "stop": 11338, "onia": 11339, "zone": 11340, "izers": 11341, "\u0120WR": 11342, "\u0120awarded": 11343, "\u0120regulatory": 11344, "\u0120Hart": 11345, "\u0120SN": 11346, "pling": 11347, "\u0120sour": 11348, "\u0120Pixel": 11349, "usive": 11350, "\u0120fet": 11351, "\u0120Sent": 11352, "\u0120automatic": 11353, "\u0120fer": 11354, "vernment": 11355, "\u0120Khan": 11356, "TON": 11357, "father": 11358, "\u0120extraordinary": 11359, "throp": 11360, "\u0120Python": 11361, "\u0120GPU": 11362, "\u0120sexually": 11363, "\u0120desktop": 11364, "itivity": 11365, "\u0120Antonio": 11366, "\u0120orient": 11367, "\u0120ears": 11368, "obby": 11369, "ouses": 11370, "vertisements": 11371, "\u0120manufacturers": 11372, "icient": 11373, "minute": 11374, "\u0120conviction": 11375, "\u0120garden": 11376, "public": 11377, "\u0120satisfied": 11378, "fold": 11379, "OK": 11380, "\u0120inhab": 11381, "\u0120Think": 11382, "\u0120programme": 11383, "\u0120stomach": 11384, "\u0120coordin": 11385, "\u0120holy": 11386, "\u0120threshold": 11387, "\u0120rhet": 11388, "\u0120serial": 11389, "\u0120employers": 11390, "\u0120Everything": 11391, "rah": 11392, "\u0120bother": 11393, "\u0120brands": 11394, "Value": 11395, "\u0120Ted": 11396, "\u0120Planet": 11397, "\u0120pink": 11398, "\u0120Furthermore": 11399, "sa": 11400, "PE": 11401, "reck": 11402, "\u0120USD": 11403, "otte": 11404, "\u0120&&": 11405, "\u0120landed": 11406, "gets": 11407, "\u0120producers": 11408, "\u0120healthcare": 11409, "\u0120dominant": 11410, "\u0120destro": 11411, "\u0120amended": 11412, "chron": 11413, "\u0120fits": 11414, "\u0120Syd": 11415, "\u0120Authority": 11416, "ATCH": 11417, "\u0120fights": 11418, "\u0120LLC": 11419, "\u0120---": 11420, "\u0120Corp": 11421, "\u0120toxic": 11422, "specific": 11423, "\u0120Corn": 11424, "\u0120Chel": 11425, "\u0120telephone": 11426, "\u0120Pant": 11427, "\u0120mysterious": 11428, "aunch": 11429, "odox": 11430, "media": 11431, "\u0120witnesses": 11432, "agu": 11433, "\u0120questioned": 11434, "\u0120Brexit": 11435, "\u0120Remember": 11436, "enez": 11437, "\u0120endorse": 11438, "iatric": 11439, "\u0120Ident": 11440, "\u0120ridiculous": 11441, "110": 11442, "\u0120prayer": 11443, "\u0120scientist": 11444, "\u01201950": 11445, "\u0120Aqu": 11446, "\u0120underground": 11447, "\u0120UFC": 11448, "mare": 11449, "\u0120Later": 11450, "wich": 11451, "\u0120subscrib": 11452, "\u0120hosts": 11453, "\u0120err": 11454, "\u0120grants": 11455, "antom": 11456, "\u0120summon": 11457, "early": 11458, "\u0120Clear": 11459, "\u0120Prim": 11460, "\u0120suspension": 11461, "\u0120guaranteed": 11462, "apper": 11463, "\u0120rice": 11464, "\u0120Sean": 11465, "\u0120Shin": 11466, "\u0120referendum": 11467, "\u0120fled": 11468, "rust": 11469, "\u0120360": 11470, "tery": 11471, "\u0120shocked": 11472, "BR": 11473, "\u0120Oil": 11474, "\u0120Allah": 11475, "\u0120partly": 11476, "\u0120ignor": 11477, "\u0120transmission": 11478, "\u0120homosexual": 11479, "iversal": 11480, "\u0120hopefully": 11481, "\u00e3\u0124\u00a4": 11482, "\u0120lesson": 11483, "Leg": 11484, "\u0120..": 11485, "Yet": 11486, "table": 11487, "appropri": 11488, "rett": 11489, "\u0120boards": 11490, "\u0120incorrect": 11491, "\u0120bacteria": 11492, "aru": 11493, "amac": 11494, "\u0120snap": 11495, ".'\"": 11496, "\u0120parad": 11497, "tem": 11498, "heart": 11499, "\u0120availability": 11500, "\u0120wisdom": 11501, "\u0120(+": 11502, "\u0120priest": 11503, "\u0120\u00c2\u0142\u0120\u00c2\u0142": 11504, "Open": 11505, "\u0120span": 11506, "\u0120parameter": 11507, "\u0120convince": 11508, "\u0120(%)": 11509, "rac": 11510, "\u0120fo": 11511, "\u0120safely": 11512, "\u0120converted": 11513, "\u0120Olympic": 11514, "\u0120reserve": 11515, "\u0120healing": 11516, "\u0120Mine": 11517, "Max": 11518, "\u0120inherent": 11519, "\u0120Graham": 11520, "\u0120integrated": 11521, "Dem": 11522, "\u0120pipeline": 11523, "\u0120applying": 11524, "\u0120embed": 11525, "\u0120Charlie": 11526, "\u0120cave": 11527, "2008": 11528, "\u0120consensus": 11529, "\u0120rewards": 11530, "Pal": 11531, "\u0120HTML": 11532, "\u0120popularity": 11533, "looking": 11534, "\u0120Sword": 11535, "\u0120Arts": 11536, "')": 11537, "\u0120electron": 11538, "clusions": 11539, "\u0120integrity": 11540, "\u0120exclusively": 11541, "\u0120grace": 11542, "\u0120torture": 11543, "\u0120burned": 11544, "two": 11545, "\u0120180": 11546, "Produ": 11547, "\u0120entreprene": 11548, "raphics": 11549, "\u0120gym": 11550, "ricane": 11551, "\u0120Tam": 11552, "\u0120administrative": 11553, "\u0120manufacturer": 11554, "\u0120vel": 11555, "\u0120Ni": 11556, "\u0120isolated": 11557, "\u0120Medicine": 11558, "\u0120backup": 11559, "\u0120promoting": 11560, "\u0120commander": 11561, "\u0120flee": 11562, "\u0120Russell": 11563, "\u0120forgotten": 11564, "\u0120Missouri": 11565, "\u0120residence": 11566, "mons": 11567, "\u0120resemb": 11568, "\u0120wand": 11569, "\u0120meaningful": 11570, "PT": 11571, "\u0120bol": 11572, "\u0120helic": 11573, "\u0120wealthy": 11574, "\u0120rifle": 11575, "strong": 11576, "rowing": 11577, "plan": 11578, "asury": 11579, "\u00e2\u0122\u00a6.": 11580, "\u0120expanding": 11581, "\u0120Hamilton": 11582, "\u0120receives": 11583, "SI": 11584, "eatures": 11585, "\u0120Anim": 11586, "REE": 11587, "Put": 11588, "\u0120briefly": 11589, "rive": 11590, "\u0120stimul": 11591, "\u0120``(": 11592, "\u0120__": 11593, "\u0120chip": 11594, "\u0120haz": 11595, "\u0120prize": 11596, "\u0120Things": 11597, "ACE": 11598, "ulin": 11599, "dict": 11600, "oku": 11601, "\u0120associate": 11602, "ockets": 11603, "youtube": 11604, "Story": 11605, "ategory": 11606, "\u0120mild": 11607, "ailing": 11608, "\u0120Ye": 11609, "Orig": 11610, "\u0120Ka": 11611, "orig": 11612, "\u0120propaganda": 11613, "\u0120anonymous": 11614, "\u0120struggled": 11615, "\u0120outrage": 11616, "ATED": 11617, "\u0120Beijing": 11618, "rary": 11619, "\u0120leather": 11620, "\u0120worlds": 11621, "\u0120broader": 11622, "125": 11623, "idal": 11624, "\u0120Better": 11625, "\u0120tear": 11626, "Ext": 11627, "\u0120proposals": 11628, "\u0120iter": 11629, "\u0120Squad": 11630, "\u0120volunt": 11631, "mi": 11632, "Did": 11633, "\u0120Pu": 11634, "pin": 11635, "\u0120speakers": 11636, "\u0120borders": 11637, "\u0120figured": 11638, "='": 11639, "\u0120simultaneously": 11640, "aeda": 11641, "\u0120charging": 11642, "\u0120urged": 11643, "\u0120conj": 11644, "256": 11645, "\u0120Gordon": 11646, "merce": 11647, "\u0120documentary": 11648, "Share": 11649, "itol": 11650, "ONE": 11651, "\u0120Garden": 11652, "hatt": 11653, "\u0120Thompson": 11654, "aneous": 11655, "apore": 11656, "\u0120tanks": 11657, "\u0120lessons": 11658, "track": 11659, "\u0120outstanding": 11660, "\u0120volunteers": 11661, "\u0120spray": 11662, "\u0120managers": 11663, "large": 11664, "\u0120camps": 11665, "\u0120artificial": 11666, "\u0120Ru": 11667, "\u0120bags": 11668, "thal": 11669, "\u0120compatible": 11670, "\u0120Blade": 11671, "\u0120fed": 11672, "\u0120argues": 11673, "FI": 11674, "\u0120unfair": 11675, "\u0120corn": 11676, "\u0120offset": 11677, "\u0120directions": 11678, "\u0120disappointed": 11679, "\u0120Convention": 11680, "\u0120viewing": 11681, "ME": 11682, "ocity": 11683, "\u0120towns": 11684, "\u0120layers": 11685, "\u0120rolled": 11686, "\u0120jumped": 11687, "\u0120attribute": 11688, "\u0120unnecess": 11689, "incoln": 11690, "\u0120suppose": 11691, "\u0120Nether": 11692, "cha": 11693, "\u0120buried": 11694, "\u0120sixth": 11695, "Ben": 11696, "ressing": 11697, "OUR": 11698, "\u0120wound": 11699, "\u0120cycl": 11700, "\u0120mechanisms": 11701, "\u0120congressional": 11702, "\u0120Element": 11703, "\u0120agreements": 11704, "\u0120decor": 11705, "\u0120closest": 11706, "\u0120Mit": 11707, "Google": 11708, "}}": 11709, "\u0120mixture": 11710, "\u0120fluid": 11711, "Sign": 11712, "\u0120Scholar": 11713, "\u0120pist": 11714, "asket": 11715, "abling": 11716, "\u0120racing": 11717, "hero": 11718, "riel": 11719, "assy": 11720, "\u0120cheaper": 11721, "ben": 11722, "\u0120vertical": 11723, "amacare": 11724, "\u0120Reading": 11725, "gments": 11726, "\u0120helicop": 11727, "\u0120sacrifice": 11728, "aya": 11729, "paren": 11730, "VA": 11731, "\u0120Les": 11732, "\u0120Studio": 11733, "\u0120violations": 11734, "\u0120Anna": 11735, "acer": 11736, "\u00e9\u00be": 11737, "\u0120Rat": 11738, "\u0120Beck": 11739, "\u0120Dick": 11740, "\u0120ACT": 11741, "\u0120composition": 11742, "\u0120texture": 11743, "\u0120Own": 11744, "\u0120smartphone": 11745, "\u0120NA": 11746, "\u0120forb": 11747, "import": 11748, "\u0120defending": 11749, "ilst": 11750, "rer": 11751, "\u0120oh": 11752, "\u0120Jeremy": 11753, "\u0120banking": 11754, "ceptions": 11755, "\u0120respective": 11756, "/.": 11757, "\u0120drinks": 11758, "\u0120Wi": 11759, "\u0120bands": 11760, "\u0120Liverpool": 11761, "\u0120grip": 11762, "\u0120Buy": 11763, "\u0120openly": 11764, "\u0120reviewed": 11765, "pert": 11766, "\u0120verify": 11767, "\u0120Cole": 11768, "\u0120Wales": 11769, "MO": 11770, "\u0120unpre": 11771, "\u0120shelter": 11772, "\u0120Imperial": 11773, "\u0120gui": 11774, "\u0120Dak": 11775, "\u0120suggestions": 11776, "\u0120explicitly": 11777, "\u0120slave": 11778, "\u0120blockchain": 11779, "\u0120competing": 11780, "\u0120promising": 11781, "SON": 11782, "\u0120soccer": 11783, "\u0120constitution": 11784, "429": 11785, "\u0120distract": 11786, "\u0120User": 11787, "esides": 11788, "\u0120Method": 11789, "\u0120Tokyo": 11790, "\u0120accompanied": 11791, "Client": 11792, "sur": 11793, "alog": 11794, "\u0120identification": 11795, "\u0120invasion": 11796, "asma": 11797, "\u0120industries": 11798, "ppers": 11799, "\u0120subtle": 11800, "\u0120Unit": 11801, "natural": 11802, "\u0120survived": 11803, "\u0120flaw": 11804, "\u013a\u0127": 11805, "\u0120Holl": 11806, "\u0120deficit": 11807, "\u0120tutorial": 11808, "\u0120Chance": 11809, "\u0120arguing": 11810, "\u0120contemporary": 11811, "\u0120integration": 11812, "forward": 11813, "\u0120tum": 11814, "itis": 11815, "\u0120hiding": 11816, "\u0120Domin": 11817, "\u0120Tan": 11818, "\u0120Building": 11819, "\u0120Vin": 11820, "\u0120spokesperson": 11821, "\u0120Notes": 11822, "\u0120emerging": 11823, "\u0120preparation": 11824, "\u0120prost": 11825, "\u0120suspects": 11826, "\u0120autonom": 11827, "Description": 11828, "\u0120dealt": 11829, "\u0120Pear": 11830, "\u0120steady": 11831, "\u0120decreased": 11832, "\u0120sovere": 11833, "\u0120Clin": 11834, "\u0120gradually": 11835, "orses": 11836, "\u0120WAR": 11837, "Serv": 11838, "\u00e3\u0124\u00a2": 11839, "hr": 11840, "\u0120dirty": 11841, "\u0120Barn": 11842, "\u0120BC": 11843, "\u0120dil": 11844, "\u0120calendar": 11845, "\u0120compliance": 11846, "\u0120chamber": 11847, "bb": 11848, "\u0120passenger": 11849, "ateful": 11850, "\u0120Title": 11851, "\u0120Sydney": 11852, "\u0120Got": 11853, "\u0120darkness": 11854, "\u0120defect": 11855, "\u0120packed": 11856, "assion": 11857, "\u0120gods": 11858, "\u0120harsh": 11859, "ICK": 11860, "leans": 11861, "\u0120algorithm": 11862, "\u0120oxygen": 11863, "\u0120visits": 11864, "\u0120blade": 11865, "\u0120kilomet": 11866, "\u0120Kentucky": 11867, "\u0120killer": 11868, "Pack": 11869, "enny": 11870, "\u0120divine": 11871, "\u0120nomination": 11872, "being": 11873, "\u0120engines": 11874, "\u0120cats": 11875, "\u0120buffer": 11876, "\u0120Phill": 11877, "\u0120traff": 11878, "AGE": 11879, "\u0120tongue": 11880, "\u0120radiation": 11881, "erer": 11882, "mem": 11883, "\u0120Explicit": 11884, "\u00e9\u00be\u012f": 11885, "\u0120couples": 11886, "\u0120physics": 11887, "\u0120McK": 11888, "\u0120politically": 11889, "awks": 11890, "\u0120Bloom": 11891, "\u0120worship": 11892, "eger": 11893, "uter": 11894, "\u0120FO": 11895, "\u0120mathemat": 11896, "\u0120sentenced": 11897, "\u0120disk": 11898, "\u0120Marg": 11899, "\u0120/*": 11900, "PI": 11901, "\u0120optional": 11902, "\u0120babies": 11903, "\u0120seeds": 11904, "\u0120Scottish": 11905, "\u0120thy": 11906, "]]": 11907, "\u0120Hitler": 11908, "PH": 11909, "ngth": 11910, "\u0120recovered": 11911, "inge": 11912, "\u0120powder": 11913, "\u0120lips": 11914, "\u0120designer": 11915, "\u0120disorders": 11916, "\u0120courage": 11917, "\u0120chaos": 11918, "\"},{\"": 11919, "\u0120carrier": 11920, "bably": 11921, "High": 11922, "\u0120RT": 11923, "esity": 11924, "len": 11925, "\u0120routes": 11926, "uating": 11927, "Fil": 11928, "NOT": 11929, "wall": 11930, "sburgh": 11931, "\u0120engaging": 11932, "\u0120JavaScript": 11933, "orer": 11934, "lihood": 11935, "\u0120unions": 11936, "\u0120Federation": 11937, "\u0120Tesla": 11938, "\u0120completion": 11939, "\u0120Ta": 11940, "\u0120privilege": 11941, "\u0120Orange": 11942, "\u0120neur": 11943, "parency": 11944, "\u0120bones": 11945, "\u0120titled": 11946, "\u0120prosecutors": 11947, "\u0120ME": 11948, "\u0120engineer": 11949, "\u0120Universe": 11950, "\u0120Hig": 11951, "nie": 11952, "oard": 11953, "\u0120hearts": 11954, "\u0120Gre": 11955, "ussion": 11956, "\u0120ministry": 11957, "\u0120penet": 11958, "\u0120Nut": 11959, "\u0120Ow": 11960, "\u0120XP": 11961, "instein": 11962, "\u0120bulk": 11963, "System": 11964, "icism": 11965, "\u0120Marketable": 11966, "\u0120preval": 11967, "\u0120poster": 11968, "\u0120attending": 11969, "urable": 11970, "\u0120licensed": 11971, "\u0120Gh": 11972, "etry": 11973, "\u0120Tradable": 11974, "\u0120blast": 11975, "\u00e0\u00a4": 11976, "\u0120Titan": 11977, "elled": 11978, "die": 11979, "Have": 11980, "\u0120Flame": 11981, "\u0120profound": 11982, "\u0120participating": 11983, "\u0120anime": 11984, "\u0120Ess": 11985, "\u0120specify": 11986, "\u0120regarded": 11987, "\u0120Spell": 11988, "\u0120sons": 11989, "owned": 11990, "\u0120merc": 11991, "\u0120experimental": 11992, "lando": 11993, "hs": 11994, "\u0120Dungeon": 11995, "inos": 11996, "\u0120comply": 11997, "\u0120Systems": 11998, "arth": 11999, "\u0120seized": 12000, "local": 12001, "\u0120Girls": 12002, "udo": 12003, "oned": 12004, "\u0120Fle": 12005, "\u0120constructed": 12006, "\u0120hosted": 12007, "\u0120scared": 12008, "actic": 12009, "\u0120Islands": 12010, "\u0120MORE": 12011, "\u0120bless": 12012, "\u0120blocking": 12013, "\u0120chips": 12014, "\u0120evac": 12015, "Ps": 12016, "\u0120corporation": 12017, "\u0120ox": 12018, "\u0120lighting": 12019, "\u0120neighbors": 12020, "\u0120Ub": 12021, "aro": 12022, "\u0120beef": 12023, "\u0120Uber": 12024, "Facebook": 12025, "armed": 12026, "itate": 12027, "\u0120Rating": 12028, "\u0120Quick": 12029, "\u0120occupied": 12030, "\u0120aims": 12031, "\u0120Additionally": 12032, "\u0120Interest": 12033, "\u0120dramatically": 12034, "\u0120heal": 12035, "\u0120painting": 12036, "\u0120engineers": 12037, "MM": 12038, "\u0120Must": 12039, "\u0120quantity": 12040, "Paul": 12041, "\u0120earnings": 12042, "\u0120Posts": 12043, "stra": 12044, "\u00e3\u0125\u00bc\u00e3\u0125": 12045, "\u0120stance": 12046, "\u0120dropping": 12047, "script": 12048, "\u0120dressed": 12049, "Make": 12050, "\u0120justify": 12051, "\u0120Ltd": 12052, "\u0120prompted": 12053, "\u0120scrut": 12054, "\u0120speeds": 12055, "\u0120Giants": 12056, "omer": 12057, "\u0120Editor": 12058, "\u0120describing": 12059, "\u0120Lie": 12060, "mented": 12061, "\u0120nowhere": 12062, "ocaly": 12063, "\u0120instruction": 12064, "fortable": 12065, "\u0120entities": 12066, "\u0120cm": 12067, "\u0120Natural": 12068, "\u0120inquiry": 12069, "\u0120pressed": 12070, "izont": 12071, "forced": 12072, "\u0120raises": 12073, "\u0120Netflix": 12074, "\u0120Side": 12075, "\u0120outer": 12076, "\u0120amongst": 12077, "ims": 12078, "owski": 12079, "\u0120climb": 12080, "never": 12081, "\u0120combine": 12082, "ding": 12083, "\u0120compr": 12084, "\u0120significance": 12085, "\u0120remembered": 12086, "\u0120Nevada": 12087, "\u0120Tel": 12088, "\u0120Scar": 12089, "\u0120Warriors": 12090, "\u0120Jane": 12091, "\u0120coup": 12092, "bas": 12093, "\u0120terminal": 12094, ",-": 12095, "OH": 12096, "\u0120tension": 12097, "\u0120wings": 12098, "\u0120Myster": 12099, "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 12100, "\u0120Unlike": 12101, "valid": 12102, "vironments": 12103, "\u0120Ali": 12104, "\u0120naked": 12105, "books": 12106, "\u0120Mun": 12107, "\u0120Gulf": 12108, "\u0120density": 12109, "\u0120dimin": 12110, "\u0120desperate": 12111, "\u0120presidency": 12112, "\u01201986": 12113, "hy": 12114, "IND": 12115, "\u0120unlock": 12116, "imens": 12117, "\u0120handled": 12118, "\u0120Eb": 12119, "\u0120disappeared": 12120, "\u0120genre": 12121, "\u01201988": 12122, "\u0120determination": 12123, "Stream": 12124, "iko": 12125, "apters": 12126, "\u0120acknowledge": 12127, "Jan": 12128, "\u0120capitalism": 12129, "Pat": 12130, "\u01202020": 12131, "\u0120painful": 12132, "\u0120curve": 12133, "\u0120bombs": 12134, "storm": 12135, "\u0120Metal": 12136, "encer": 12137, "\u0120Fig": 12138, "\u0120Aaron": 12139, "anches": 12140, "\u0120inspiration": 12141, "\u0120exhaust": 12142, "tains": 12143, "ashi": 12144, "\u0120descript": 12145, "\u0120ritual": 12146, "\u0120Chelsea": 12147, "\u0120promotion": 12148, "\u0120Hung": 12149, "\u0120Ward": 12150, "iva": 12151, "\u0120ET": 12152, "\u0120toss": 12153, "allow": 12154, "\u0120Francis": 12155, "Dep": 12156, "\u0120happiness": 12157, "\u0120Glass": 12158, "\u0120beta": 12159, "\u0120strengthen": 12160, "NE": 12161, "oa": 12162, "\u0120buttons": 12163, "\u0120Murray": 12164, "\u0120kicked": 12165, "Quest": 12166, "\u0120Talk": 12167, "\u0120Several": 12168, "\u0120Zero": 12169, "\u0120drone": 12170, "ulk": 12171, "\u0120cam": 12172, "\u0120Mobile": 12173, "\u0120preventing": 12174, "\u0120retro": 12175, "\u0120Ax": 12176, "\u0120cruel": 12177, "\u0120float": 12178, ".),": 12179, "\u0120filing": 12180, "\u0120Grant": 12181, "\u0120Bor": 12182, "\u0120rib": 12183, "\u0120championship": 12184, "\u0120Merc": 12185, "\u0120styles": 12186, "\u0120cake": 12187, "\u0120builds": 12188, "\u0120Self": 12189, "iox": 12190, "\u0120epic": 12191, "oyd": 12192, "Bel": 12193, "\u0120Stew": 12194, ".(": 12195, "ahu": 12196, "\u0120Beyond": 12197, "\u0120outs": 12198, "\u0120solo": 12199, "\u0120Tree": 12200, "\u0120preserve": 12201, "\u0120tub": 12202, "ARE": 12203, "roc": 12204, "\u0120Impro": 12205, "\u0120Wright": 12206, "\u0120bund": 12207, "\u0120traged": 12208, "\u0120occasional": 12209, "bian": 12210, "Second": 12211, "rons": 12212, "\u0120interactions": 12213, "formed": 12214, "sing": 12215, "\u0120owns": 12216, "\u0120hockey": 12217, "General": 12218, "\u0120logical": 12219, "\u0120expend": 12220, "\u0120escal": 12221, "\u0120Griff": 12222, "\u0120Crown": 12223, "\u0120Reserve": 12224, "\u0120stopping": 12225, "\u0120excuse": 12226, "second": 12227, "\u0120operated": 12228, "\u0120reaches": 12229, "\u0120Malays": 12230, "\u0120pollution": 12231, "\u0120Brooklyn": 12232, "\u0120delete": 12233, "\u0120hash": 12234, "Block": 12235, "aha": 12236, "\u00e2\u0122\u00b3": 12237, "\u0120shorter": 12238, "piece": 12239, ">>>": 13163, "\u0120Mormon": 13164, "tor": 13165, "\u0120particles": 13166, "\u0120Bart": 13167, "ryption": 13168, "\u0120admin": 13169, "\u0120squee": 13170, "VIDIA": 13171, "\u0120creator": 13172, "iameter": 13173, "icular": 13174, "NBC": 13175, "\u0120grabbed": 13176, "\u0120nodd": 13177, "\u0120rated": 13178, "\u0120rotation": 13179, "\u0120grasp": 13180, "\u0120excessive": 13181, "\u0120EC": 13182, "\u0120Whit": 13183, "\u0120inventory": 13184, "aults": 13185, "\u0120FB": 13186, "\u0120ecosystem": 13187, "\u0120billions": 13188, "\u0120venture": 13189, "named": 13190, "\u0120defender": 13191, "oute": 13192, "Instead": 13193, "irable": 13194, "War": 13195, "\u0120assumption": 13196, "\u0120bite": 13197, "\u0120earthqu": 13198, "tail": 13199, "space": 13200, "\u0120gifts": 13201, "boys": 13202, "\u0120inevitable": 13203, "\u0120structural": 13204, "\u0120beneficial": 13205, "\u0120compelling": 13206, "hole": 13207, "ervation": 13208, "\u0120coat": 13209, "oj": 13210, "incarn": 13211, "\u0120Years": 13212, "\u0120determining": 13213, "\u0120rhetoric": 13214, "\u0120boundaries": 13215, "\u0120whites": 13216, "Ant": 13217, "addy": 13218, ")-": 13219, "raham": 13220, "etermin": 13221, "\u0120harvest": 13222, "\u0120Conc": 13223, "\u0120laptop": 13224, "\u0120Match": 13225, "\u0120enjoying": 13226, "cca": 13227, "ollar": 13228, "\u0120trips": 13229, "\u0120addiction": 13230, "\u0120Sak": 13231, "\u0120powered": 13232, "\u0120cous": 13233, "\u0120Russians": 13234, "iere": 13235, "\u0120retrie": 13236, "quality": 13237, "\u0120differ": 13238, "\u0120kingdom": 13239, "\u0120Laur": 13240, "\u0120Capitol": 13241, "\u0120conclusions": 13242, "\u0120Altern": 13243, "\u0120Nav": 13244, "\u0120transparent": 13245, "BER": 13246, "Group": 13247, "\u0120Complete": 13248, "\u0120infer": 13249, "\u0120intrig": 13250, "\u0120insane": 13251, "RO": 13252, "ophob": 13253, "isen": 13254, "qual": 13255, "Michael": 13256, "\u0120museum": 13257, "\u0120Pope": 13258, "\u0120reset": 13259, "rative": 13260, "five": 13261, "\u0120aggreg": 13262, "ittees": 13263, "ository": 13264, "\u0120carb": 13265, "\u0120Record": 13266, "\u0120decides": 13267, "\u0120Fix": 13268, "\u0120exceptions": 13269, "\u0120Commissioner": 13270, "uns": 13271, "\u0120Environmental": 13272, "\u0120legendary": 13273, "istence": 13274, "\u0120tunnel": 13275, "km": 13276, "\u0120insult": 13277, "\u0120troll": 13278, "\u0120shake": 13279, "\u0120detention": 13280, "ques": 13281, "\u0120Chrome": 13282, "\u0120Files": 13283, "\u0120subt": 13284, "\u0120prospects": 13285, "\u0120prol": 13286, "render": 13287, "proof": 13288, "\u0120performances": 13289, "Str": 13290, "\u0120href": 13291, "ername": 13292, "\u0120achievement": 13293, "\u0120fut": 13294, "Full": 13295, "\u0120Leban": 13296, "google": 13297, "\u00e3\u0125\u012a": 13298, "ampa": 13299, "Maybe": 13300, "\u0120projected": 13301, "\u0120Emb": 13302, "\u0120colleg": 13303, "\u0120awards": 13304, "\u0120\u00e2\u0136": 13305, "Gold": 13306, "\u0120Blake": 13307, "\u0120Raj": 13308, "ifting": 13309, "\u0120pending": 13310, "\u0120instinct": 13311, "\u0120developments": 13312, "Connect": 13313, "\u0120Mand": 13314, "\u0120WITH": 13315, "\u0120Philippines": 13316, "profile": 13317, "\u0120altogether": 13318, "\u0120Bund": 13319, "\u0120TD": 13320, "oooo": 13321, "amped": 13322, "iph": 13323, "\u0120steam": 13324, "\u0120oldest": 13325, "\u0120detection": 13326, "ulpt": 13327, "\u0120\u00e7": 13328, "\u0120Wayne": 13329, "2006": 13330, "fa": 13331, "\u0120circles": 13332, "\u0120Fu": 13333, "\u0120donors": 13334, "appropriate": 13335, "\u0120Dakota": 13336, "jamin": 13337, "\u0120motivated": 13338, "\u0120purchases": 13339, "\u0120Louisiana": 13340, "\u0120Spl": 13341, "\u0120globe": 13342, "\u0120105": 13343, "zip": 13344, "call": 13345, "\u0120departments": 13346, "\u0120sustainable": 13347, "105": 13348, "\u0120OP": 13349, "ifiers": 13350, "\u0120prevented": 13351, "\u0120incomp": 13352, "\u0120Commander": 13353, "\u0120dominated": 13354, "\u0120\u00c2\u00bb": 13355, "\u0120invested": 13356, "\u0120complexity": 13357, "\u0120incl": 13358, "\u0120ensuring": 13359, "\u0120realm": 13360, "ync": 13361, "\u0120Independent": 13362, "rained": 13363, "\u0120Jen": 13364, "\u0120Flight": 13365, "\u0120athe": 13366, "\u0120speculation": 13367, "\u0120TE": 13368, "ocate": 13369, "tic": 13370, "\u0120plaint": 13371, "herry": 13372, "\u0120toy": 13373, "\u0120111": 13374, "\u0120plates": 13375, "status": 13376, "\u0120Isa": 13377, "\u0120devoted": 13378, "Cop": 13379, "\u0120ES": 13380, "255": 13381, "urrency": 13382, "Main": 13383, "\u0120slaves": 13384, "\u0120pepper": 13385, "\u0120quotes": 13386, "\u0120ceiling": 13387, "\u0120Fish": 13388, "\u0120transformation": 13389, "\u0120fraction": 13390, "\u0120advantages": 13391, "\u0120toile": 13392, "\u0120stunning": 13393, "\u0120moist": 13394, "breaking": 13395, "si": 13396, "\u0120Location": 13397, "\u0120Medium": 13398, "\u0120texts": 13399, "\u0120ugly": 13400, "\u0120bio": 13401, ".\u00e2\u0122\u0136": 13402, "\u0120Based": 13403, "\u0120trains": 13404, "\u0120Wing": 13405, "\u0120Ancient": 13406, "\u0120Records": 13407, "\u0120Hope": 13408, "Special": 13409, "adesh": 13410, "obi": 13411, "[/": 13412, "\u0120temporarily": 13413, "Ver": 13414, "hu": 13415, "oser": 13416, "\u0120overnight": 13417, "\u0120mamm": 13418, "\u0120Treasury": 13419, "\u0120Venezuel": 13420, "\u0120Mega": 13421, "\u0120tar": 13422, "\u0120expects": 13423, "black": 13424, "orph": 13425, "\\\\\\\\": 13426, "\u0120acceptance": 13427, "\u0120radar": 13428, "sis": 13429, "\u0120junior": 13430, "\u0120frames": 13431, "\u0120observation": 13432, "acies": 13433, "Power": 13434, "\u0120Advanced": 13435, "Mag": 13436, "ologically": 13437, "\u0120Mechan": 13438, "\u0120sentences": 13439, "\u0120analysts": 13440, "aughters": 13441, "forcement": 13442, "\u0120vague": 13443, "\u0120clause": 13444, "\u0120directors": 13445, "\u0120evaluate": 13446, "\u0120cabinet": 13447, "Matt": 13448, "\u0120Classic": 13449, "Ang": 13450, "\u0120cler": 13451, "\u0120Buck": 13452, "\u0120researcher": 13453, "\u0120160": 13454, "\u0120poorly": 13455, "\u0120experiencing": 13456, "\u0120Ped": 13457, "\u0120Manhattan": 13458, "\u0120freed": 13459, "\u0120themes": 13460, "advant": 13461, "\u0120nin": 13462, "\u0120praise": 13463, "104": 13464, "\u0120Libya": 13465, "best": 13466, "\u0120trusted": 13467, "\u0120cease": 13468, "\u0120dign": 13469, "Direct": 13470, "\u0120bombing": 13471, "\u0120migration": 13472, "\u0120Sciences": 13473, "\u0120municipal": 13474, "\u0120Average": 13475, "\u0120glory": 13476, "\u0120revealing": 13477, "\u0120arena": 13478, "\u0120uncertainty": 13479, "\u0120battlefield": 13480, "iao": 13481, "God": 13482, "\u0120cinem": 13483, "rape": 13484, "elle": 13485, "apons": 13486, "\u0120listing": 13487, "\u0120waited": 13488, "\u0120spotted": 13489, "keley": 13490, "\u0120Audio": 13491, "eor": 13492, "arding": 13493, "idding": 13494, "igma": 13495, "\u0120Neg": 13496, "\u0120lone": 13497, "\u0120----": 13498, "exe": 13499, "deg": 13500, "\u0120transf": 13501, "\u0120wash": 13502, "\u0120slavery": 13503, "\u0120exploring": 13504, "\u0120WW": 13505, "atson": 13506, "\u0120encl": 13507, "lies": 13508, "\u0120Creek": 13509, "\u0120wooden": 13510, "Manager": 13511, "\u0120Brand": 13512, "ummy": 13513, "\u0120Arthur": 13514, "\u0120bureaucr": 13515, "\u0120blend": 13516, "arians": 13517, "Further": 13518, "\u0120supposedly": 13519, "\u0120winds": 13520, "\u01201979": 13521, "\u0120gravity": 13522, "\u0120analyses": 13523, "\u0120Travel": 13524, "\u0120Veter": 13525, "\u0120dumb": 13526, "\u0120alternate": 13527, "gal": 13528, "\u0120consumed": 13529, "\u0120effectiveness": 13530, ".''": 13531, "\u0120paths": 13532, "onda": 13533, "LA": 13534, "\u0120Strong": 13535, "\u0120enables": 13536, "\u0120escaped": 13537, "\u0120\"\"": 13538, "\u0120112": 13539, "\u01201983": 13540, "\u0120smiled": 13541, "\u0120tendency": 13542, "Fire": 13543, "\u0120pars": 13544, "\u0120Roc": 13545, "\u0120lake": 13546, "\u0120fitness": 13547, "\u0120Ath": 13548, "\u0120Horn": 13549, "\u0120hier": 13550, "\u0120impose": 13551, "mother": 13552, "\u0120pension": 13553, "icut": 13554, "borne": 13555, "iciary": 13556, "._": 13557, "\u0120SU": 13558, "\u0120polar": 13559, "isy": 13560, "engu": 13561, "itialized": 13562, "ATA": 13563, "write": 13564, "\u0120exercises": 13565, "\u0120Diamond": 13566, "otypes": 13567, "\u0120harmful": 13568, "onz": 13569, "\u0120printing": 13570, "story": 13571, "\u0120expertise": 13572, "\u0120Ger": 13573, "\u0120tragedy": 13574, "\u0120Fly": 13575, "\u0120divid": 13576, "ampire": 13577, "stock": 13578, "Mem": 13579, "\u0120reign": 13580, "\u0120unve": 13581, "\u0120amend": 13582, "\u0120Prophet": 13583, "\u0120mutual": 13584, "\u0120Fac": 13585, "\u0120replacing": 13586, "Har": 13587, "\u0120Circuit": 13588, "\u0120throat": 13589, "\u0120Shot": 13590, "\u0120batteries": 13591, "\u0120toll": 13592, "\u0120addressing": 13593, "\u0120Medicaid": 13594, "\u0120pupp": 13595, "\u0120Nar": 13596, "olk": 13597, "\u0120equity": 13598, "MR": 13599, "\u0120Hispan": 13600, "\u0120Large": 13601, "mid": 13602, "Dev": 13603, "\u0120exped": 13604, "\u0120demo": 13605, "\u0120Marshall": 13606, "ergus": 13607, "\u0120fiber": 13608, "\u0120divorce": 13609, "\u0120Create": 13610, "\u0120slower": 13611, "\u0120Parker": 13612, "\u0120Student": 13613, "\u0120Training": 13614, "Return": 13615, "\u0120Tru": 13616, "\u0120cub": 13617, "\u0120Reached": 13618, "\u0120panic": 13619, "\u0120quarters": 13620, "\u0120rect": 13621, "\u0120treating": 13622, "\u0120rats": 13623, "\u0120Christianity": 13624, "oler": 13625, "\u0120sacred": 13626, "\u0120declare": 13627, "ulative": 13628, "eting": 13629, "\u0120delivering": 13630, "estone": 13631, "\u0120tel": 13632, "\u0120Larry": 13633, "\u0120meta": 13634, "accept": 13635, "artz": 13636, "\u0120Roger": 13637, "handed": 13638, "\u0120header": 13639, "\u0120trapped": 13640, "\u0120Century": 13641, "\u0120knocked": 13642, "\u0120Oxford": 13643, "\u0120survivors": 13644, "bot": 13645, "\u0120demonstration": 13646, "\u0120dirt": 13647, "\u0120assists": 13648, "OME": 13649, "\u0120Draft": 13650, "ortunate": 13651, "folio": 13652, "pered": 13653, "usters": 13654, "gt": 13655, "\u0120Lock": 13656, "\u0120judicial": 13657, "verted": 13658, "\u0120secured": 13659, "outing": 13660, "\u0120Books": 13661, "\u0120hosting": 13662, "\u0120lifted": 13663, "length": 13664, "\u0120jer": 13665, "\u0120wheels": 13666, "\u0120Range": 13667, "umbnails": 13668, "\u0120diagnosis": 13669, "tech": 13670, "\u0120Stewart": 13671, "\u0120Pract": 13672, "\u0120nationwide": 13673, "\u0120dear": 13674, "\u0120obligations": 13675, "\u0120grows": 13676, "\u0120mandatory": 13677, "\u0120suspicious": 13678, "!'": 13679, "Apr": 13680, "Great": 13681, "\u0120mortgage": 13682, "\u0120prosecutor": 13683, "\u0120editorial": 13684, "\u0120Kr": 13685, "\u0120processed": 13686, "ungle": 13687, "\u0120flexibility": 13688, "Earlier": 13689, "\u0120Cart": 13690, "\u0120Sug": 13691, "\u0120focuses": 13692, "\u0120startup": 13693, "\u0120breach": 13694, "\u0120Tob": 13695, "cycle": 13696, "\u00e3\u0122\u012e": 13697, "rose": 13698, "\u0120bizarre": 13699, "\u00e3\u0122\u012f": 13700, "\u0120vegetables": 13701, "$$": 13702, "\u0120retreat": 13703, "oshi": 13704, "\u0120Shop": 13705, "\u0120Ground": 13706, "\u0120Stop": 13707, "\u0120Hawaii": 13708, "\u0120Ay": 13709, "Perhaps": 13710, "\u0120Beaut": 13711, "uffer": 13712, "enna": 13713, "\u0120productivity": 13714, "Fixed": 13715, "control": 13716, "\u0120absent": 13717, "\u0120Campaign": 13718, "Green": 13719, "\u0120identifying": 13720, "\u0120regret": 13721, "\u0120promoted": 13722, "\u0120Seven": 13723, "\u0120eru": 13724, "neath": 13725, "aughed": 13726, "\u0120Pin": 13727, "\u0120Living": 13728, "Cost": 13729, "omatic": 13730, "mega": 13731, "\u0120Nig": 13732, "ocy": 13733, "\u0120inbox": 13734, "\u0120empire": 13735, "\u0120horizont": 13736, "\u0120branches": 13737, "\u0120metaph": 13738, "Active": 13739, "edi": 13740, "\u0120Film": 13741, "\u0120Something": 13742, "\u0120mods": 13743, "incial": 13744, "\u0120Original": 13745, "Gen": 13746, "\u0120spirits": 13747, "\u0120earning": 13748, "Hist": 13749, "\u0120riders": 13750, "\u0120sacrific": 13751, "MT": 13752, "\u0120VA": 13753, "\u0120Salt": 13754, "\u0120occupation": 13755, "\u0120Mi": 13756, "\u0120disg": 13757, "lict": 13758, "\u0120nit": 13759, "\u0120nodes": 13760, "eem": 13761, "\u0120Pier": 13762, "\u0120hatred": 13763, "psy": 13764, "\u00e3\u0125\u012b": 13765, "\u0120theater": 13766, "\u0120sophisticated": 13767, "\u0120defended": 13768, "\u0120besides": 13769, "\u0120thoroughly": 13770, "\u0120Medicare": 13771, "\u0120blamed": 13772, "arently": 13773, "\u0120crying": 13774, "FOR": 13775, "priv": 13776, "\u0120singing": 13777, "\u0120Il": 13778, "\u0120cute": 13779, "oided": 13780, "olitical": 13781, "\u0120Neuro": 13782, "\u00e5\u00a4": 13783, "\u0120donation": 13784, "\u0120Eagles": 13785, "\u0120Give": 13786, "Tom": 13787, "\u0120substantially": 13788, "\u0120License": 13789, "\u0120Ja": 13790, "\u0120grey": 13791, "\u0120Animal": 13792, "\u0120ER": 13793, "\u0120Und": 13794, "\u0120keen": 13795, "\u0120conclude": 13796, "\u0120Mississippi": 13797, "Engine": 13798, "\u0120Studios": 13799, "Press": 13800, "overs": 13801, "llers": 13802, "\u0120350": 13803, "\u0120Rangers": 13804, "\u0120rou": 13805, "erto": 13806, "Ep": 13807, "issa": 13808, "ivan": 13809, "\u0120seal": 13810, "\u0120Regist": 13811, "display": 13812, "\u0120weaken": 13813, "uum": 13814, "\u0120Commons": 13815, "\u0120Say": 13816, "\u0120cultures": 13817, "\u0120laughed": 13818, "\u0120slip": 13819, "\u0120treatments": 13820, "izable": 13821, "mart": 13822, "\u0120Rice": 13823, "\u0120beast": 13824, "\u0120obesity": 13825, "\u0120Laure": 13826, "iga": 13827, "Which": 13828, "holder": 13829, "\u0120elderly": 13830, "\u0120pays": 13831, "\u0120complained": 13832, "\u0120crop": 13833, "\u0120proc": 13834, "\u0120explosive": 13835, "\u0120Fan": 13836, "\u0120Arsenal": 13837, "Author": 13838, "eful": 13839, "\u0120meals": 13840, "\u0120(-": 13841, "idays": 13842, "\u0120imagination": 13843, "\u0120annually": 13844, "\u0120ms": 13845, "asures": 13846, "Head": 13847, "ikh": 13848, "matic": 13849, "\u0120boyfriend": 13850, "\u0120Computer": 13851, "\u0120bump": 13852, "\u0120surge": 13853, "\u0120Craig": 13854, "\u0120Kirk": 13855, "Del": 13856, "mediate": 13857, "\u0120scenarios": 13858, "\u0120Mut": 13859, "\u0120Stream": 13860, "\u0120competitors": 13861, "\u00d9\u0126": 13862, "\u0120Stanford": 13863, "\u0120Resources": 13864, "azed": 13865, "bage": 13866, "\u0120organis": 13867, "\u0120Release": 13868, "\u0120separately": 13869, "\u0120habits": 13870, "\u0120measurements": 13871, "\u0120Close": 13872, "\u0120accompany": 13873, "\u0120gly": 13874, "\u0120tang": 13875, "\u0120Rou": 13876, "\u0120plugin": 13877, "\u0120convey": 13878, "\u0120Challenge": 13879, "oots": 13880, "jan": 13881, "\u0120curs": 13882, "\u0120Relations": 13883, "keeper": 13884, "\u0120approaching": 13885, "ping": 13886, "Speaking": 13887, "\u0120arrangement": 13888, "\u0120VI": 13889, "arettes": 13890, "\u0120affecting": 13891, "\u0120permits": 13892, "because": 13893, "\u0120useless": 13894, "\u0120Hus": 13895, "!!!!": 13896, "\u0120destroying": 13897, "Unfortunately": 13898, "\u0120fascinating": 13899, "Sem": 13900, "\u0120electoral": 13901, "\u0120transparency": 13902, "\u0120Chaos": 13903, "\u0120volunteer": 13904, "\u0120statistical": 13905, "\u0120activated": 13906, "rox": 13907, "Web": 13908, "HE": 13909, "\u0120Hampshire": 13910, "isive": 13911, "Map": 13912, "\u0120trash": 13913, "\u0120Lawrence": 13914, "stick": 13915, "Cr": 13916, "\u0120rings": 13917, "EXT": 13918, "\u0120operational": 13919, "opes": 13920, "Does": 13921, "\u0120Evans": 13922, "\u0120witnessed": 13923, "Port": 13924, "\u0120launching": 13925, "econom": 13926, "wear": 13927, "\u0120Particip": 13928, "umm": 13929, "cules": 13930, "\u0120RAM": 13931, "\u0120Tun": 13932, "\u0120assured": 13933, "\u0120binary": 13934, "\u0120betray": 13935, "\u0120exploration": 13936, "\u0120Fel": 13937, "\u0120admission": 13938, "itated": 13939, "Sy": 13940, "\u0120avoided": 13941, "\u0120Simulator": 13942, "\u0120celebrated": 13943, "\u0120Electric": 13944, "\u00a5\u0140": 13945, "\u0120cluster": 13946, "itzerland": 13947, "health": 13948, "Line": 13949, "\u0120Nash": 13950, "aton": 13951, "\u0120spare": 13952, "\u0120enterprise": 13953, "\u0120DIS": 13954, "cludes": 13955, "\u0120flights": 13956, "\u0120regards": 13957, "\u0120\u00c3\u0139": 13958, "half": 13959, "\u0120trucks": 13960, "\u0120contacts": 13961, "\u0120uncons": 13962, "\u0120Climate": 13963, "\u0120immense": 13964, "NEW": 13965, "occ": 13966, "ective": 13967, "\u0120embod": 13968, "\u0120patrol": 13969, "\u0120beside": 13970, "\u0120viable": 13971, "\u0120creep": 13972, "\u0120triggered": 13973, "verning": 13974, "\u0120comparable": 13975, "ql": 13976, "\u0120gaining": 13977, "asses": 13978, "\u0120();": 13979, "\u0120Grey": 13980, "\u0120MLS": 13981, "sized": 13982, "\u0120prosper": 13983, "\"?": 13984, "\u0120polling": 13985, "\u0120shar": 13986, "\u0120RC": 13987, "\u0120firearm": 13988, "orient": 13989, "\u0120fence": 13990, "\u0120variations": 13991, "giving": 13992, "\u0120Pi": 13993, "ospel": 13994, "\u0120pledge": 13995, "\u0120cure": 13996, "\u0120spy": 13997, "\u0120violated": 13998, "\u0120rushed": 13999, "\u0120stroke": 14000, "\u0120Blog": 14001, "sels": 14002, "\u0120Ec": 14003, ",''": 14004, "\u0120pale": 14005, "\u0120Collins": 14006, "terror": 14007, "\u0120Canadians": 14008, "\u0120tune": 14009, "\u0120laboratory": 14010, "\u0120nons": 14011, "tarian": 14012, "\u0120disability": 14013, "\u0120Gam": 14014, "\u0120singer": 14015, "alg": 14016, "\u0120Senior": 14017, "\u0120traded": 14018, "\u0120Warrior": 14019, "\u0120infring": 14020, "\u0120Franklin": 14021, "\u0120strain": 14022, "\u0120Swedish": 14023, "\u0120seventh": 14024, "\u0120Benn": 14025, "\u0120Tell": 14026, "\u0120syndrome": 14027, "\u0120wondered": 14028, "iden": 14029, "++++": 14030, "igo": 14031, "\u0120purple": 14032, "\u0120journalism": 14033, "\u0120rebel": 14034, "\u0120fu": 14035, "blog": 14036, "\u0120invite": 14037, "rencies": 14038, "\u0120Contact": 14039, "Israel": 14040, "\u0120Content": 14041, "\u0120cheer": 14042, "\u0120bedroom": 14043, "\u0120Engineering": 14044, "\u0120Queens": 14045, "\u0120dwell": 14046, "\u0120PlayStation": 14047, "\u0120Dim": 14048, "\u0120Colon": 14049, "lr": 14050, "\u0120operates": 14051, "\u0120motivation": 14052, "USA": 14053, "astered": 14054, "Core": 14055, "\u0120Truth": 14056, "olo": 14057, "OSE": 14058, "\u0120Memory": 14059, "\u0120predec": 14060, "\u0120anarch": 14061, "\u01201920": 14062, "\u0120Yam": 14063, "\u00c3\u00a8": 14064, "bid": 14065, "\u0120grateful": 14066, "\u0120excitement": 14067, "\u0120treasure": 14068, "\u0120longest": 14069, "ctive": 14070, "\u0120deserves": 14071, "\u0120reserves": 14072, "\u0120cops": 14073, "\u0120Ottawa": 14074, "\u0120Egyptian": 14075, "anked": 14076, "\u0120artif": 14077, "\u0120hypothesis": 14078, ":/": 14079, "\u0120purchasing": 14080, "\u0120lovely": 14081, "HP": 14082, "\u0120divide": 14083, "\u0120strictly": 14084, "\u0120questioning": 14085, "\u0120taxpayers": 14086, "\u0120Joy": 14087, "\u0120rolls": 14088, "\u0120Heavy": 14089, "\u0120ports": 14090, "\u0120magnetic": 14091, "\u0120inflamm": 14092, "\u0120brush": 14093, "tics": 14094, "\u00e2\u012a\u0134": 14095, "\u0120bottles": 14096, "ppy": 14097, "\u0120padd": 14098, "\u00e3\u0124\u00af": 14099, "million": 14100, "\u0120devastating": 14101, "\u0120compiled": 14102, "\u0120medication": 14103, "\u0120twelve": 14104, "\u0120Perry": 14105, "Space": 14106, "imb": 14107, "your": 14108, "\u0120leaked": 14109, "\u0120Tar": 14110, "\u0120unity": 14111, "\u0120infected": 14112, "\u0120traveled": 14113, "IDE": 14114, "\u0120McDonald": 14115, "txt": 14116, "\u0120Princ": 14117, "\u0120interven": 14118, "\u0120Taiwan": 14119, "\u0120Pow": 14120, "\u0120bearing": 14121, "\u0120Thread": 14122, "\u0120zones": 14123, "izards": 14124, "unks": 14125, "Chapter": 14126, "llor": 14127, "\u0120\u00c2\u00b7": 14128, "\u0120wounds": 14129, "\u0120discretion": 14130, "\u0120succeeded": 14131, "iking": 14132, "\u0120iconic": 14133, "Call": 14134, "\u0120screening": 14135, "\u0120Mis": 14136, "icts": 14137, "\u0120ministers": 14138, "\u0120separation": 14139, "Player": 14140, "\u0120bip": 14141, "\u0120beloved": 14142, "\u0120counting": 14143, "\u0120Eye": 14144, "around": 14145, "inging": 14146, "\u0120tablet": 14147, "\u0120offence": 14148, "inance": 14149, "have": 14150, "\u0120Info": 14151, "\u0120Ninja": 14152, "\u0120protective": 14153, "\u0120Cass": 14154, "Mac": 14155, "\u0120Quality": 14156, "North": 14157, "\u0120ic": 14158, "\u0120Cuba": 14159, "\u0120Chronicle": 14160, "\u0120Property": 14161, "\u0120fastest": 14162, "otos": 14163, "\u0120Germ": 14164, "OWN": 14165, "\u0120boom": 14166, "\u0120Stanley": 14167, "erguson": 14168, "\u0120clever": 14169, "\u0120enters": 14170, "mode": 14171, "terior": 14172, "\u0120Sens": 14173, "\u0120linear": 14174, "ARK": 14175, "\u0120comparing": 14176, "\u0120purely": 14177, "\u0120safer": 14178, "\u0120Potter": 14179, "\u0120cups": 14180, "RT": 14181, "\u0120gluc": 14182, "\u0120attributed": 14183, "\u0120dupl": 14184, "\u0120Pap": 14185, "\u0120precious": 14186, "\u0120pa": 14187, "ictionary": 14188, "\u0120Tig": 14189, "\u0120Too": 14190, "olutions": 14191, "stan": 14192, "\u0120robots": 14193, "\u0120lobb": 14194, "\u0120statute": 14195, "\u0120prevention": 14196, "western": 14197, "160": 14198, "\u0120Active": 14199, "\u0120Maria": 14200, "hal": 14201, "None": 14202, "ellar": 14203, "\u0120KB": 14204, "\u0120Partners": 14205, "\u0120Single": 14206, "\u0120Following": 14207, "ango": 14208, "acious": 14209, "\u0120thou": 14210, "\u0120kg": 14211, "\u0120influential": 14212, "\u0120Friends": 14213, "Sur": 14214, "ainted": 14215, "\u0120forums": 14216, "\u0120starter": 14217, "\u0120citizenship": 14218, "\u0120Election": 14219, "onge": 14220, "otation": 14221, "osph": 14222, ";;;;": 14223, "utical": 14224, "pur": 14225, "eren": 14226, "\u0120accusations": 14227, "bitious": 14228, "abbit": 14229, "\u0120Ord": 14230, "Posted": 14231, "irk": 14232, "\u0120sensitivity": 14233, "iche": 14234, "\u0120Amy": 14235, "\u0120Fab": 14236, "\u0120summit": 14237, "\u0120pedest": 14238, "\u0120rubber": 14239, "\u0120agricultural": 14240, "\u0120cancel": 14241, "AE": 14242, "\u0120inaug": 14243, "\u0120contam": 14244, "\u0120firmly": 14245, "iw": 14246, "stage": 14247, "\u0120Kan": 14248, "\u0120tier": 14249, "\u0120invention": 14250, "\u0120translated": 14251, "\u0120Rules": 14252, "Box": 14253, "Twitter": 14254, "IDS": 14255, "\u0120pizza": 14256, "\u0120debug": 14257, "\u0120Drop": 14258, "vs": 14259, "\u0120horses": 14260, "big": 14261, "\u0120boring": 14262, "\u0120hood": 14263, "\u0120McCain": 14264, "atched": 14265, "\u0120Bros": 14266, "\u0120skip": 14267, "\u0120essay": 14268, "stat": 14269, "\u0120Legends": 14270, "\u0120ammunition": 14271, "auc": 14272, "\u0120shooter": 14273, "\u0120unh": 14274, "\u0120supplied": 14275, "\u0120generic": 14276, "\u0120SK": 14277, "iban": 14278, "yrics": 14279, "\u0120255": 14280, "\u0120climbing": 14281, "Former": 14282, "\u0120flip": 14283, "\u0120jumping": 14284, "\u0120frustration": 14285, "\u0120Terry": 14286, "\u0120neighborhoods": 14287, "\u0120median": 14288, "bean": 14289, "\u0120brains": 14290, "Following": 14291, "\u0120shaped": 14292, "\u0120draws": 14293, "\u0120altered": 14294, "Jack": 14295, "\u0120recipes": 14296, "\u0120skilled": 14297, "wealth": 14298, "achi": 14299, "election": 14300, "\u0120behaviors": 14301, "deals": 14302, "\u0120Until": 14303, "Fe": 14304, "\u0120declaration": 14305, "marks": 14306, "\u0120Between": 14307, "celona": 14308, "\u0120reson": 14309, "\u0120bubble": 14310, "Among": 14311, "\u0120imperial": 14312, "GS": 14313, "\u0120feminist": 14314, "2005": 14315, "\u0120Kyle": 14316, "\u0120accounting": 14317, "\u0120Tele": 14318, "\u0120Tyr": 14319, "\u0120connecting": 14320, "\u0120rehab": 14321, "\u0120Pred": 14322, "sim": 14323, "\u0120meantime": 14324, "\u0120physician": 14325, "MW": 14326, "\u0120Campbell": 14327, "\u0120Brandon": 14328, "\u0120contributing": 14329, "\u0120Rule": 14330, "\u0120Weight": 14331, "\u0120Nap": 14332, "\u0120interactive": 14333, "\u0120vag": 14334, "\u0120helmet": 14335, "\u0120Comb": 14336, "four": 14337, "\u0120shipped": 14338, "\u0120completing": 14339, "\u0120PD": 14340, "PDATE": 14341, "\u0120spreading": 14342, "\u0120scary": 14343, "erving": 14344, "\u0120Gas": 14345, "\u0120frank": 14346, "school": 14347, "\u0120romantic": 14348, "\u0120stabil": 14349, "Rob": 14350, "\u0120accurately": 14351, "\u0120acute": 14352, "\u0120Hann": 14353, "\u0120symbols": 14354, "\u0120civilization": 14355, "\u0120AW": 14356, "\u0120lightning": 14357, "\u0120considers": 14358, "\u0120venue": 14359, "\u0120\u00d7": 14360, "\u0120oven": 14361, "\u0120SF": 14362, "his": 14363, "\u0120nu": 14364, "\u0120Learn": 14365, "\u0120peoples": 14366, "\u0120std": 14367, "\u0120slee": 14368, "\u0120slic": 14369, "\u0120Statistics": 14370, "\u0120corners": 14371, "\u0120Baker": 14372, "\u0120:)": 14373, "mentation": 14374, "olver": 14375, "\u0120laughing": 14376, "\u0120Todd": 14377, "onde": 14378, "\u0120Hills": 14379, "\u0120nuts": 14380, "\u0120Woman": 14381, "plane": 14382, "\u0120liver": 14383, "\u0120Inside": 14384, "Sorry": 14385, "\u0120agrees": 14386, "\u0120fundament": 14387, "\u0120Fisher": 14388, "\u0120auction": 14389, "\u0120threads": 14390, "glas": 14391, "\u0120Basic": 14392, "\u0120Nat": 14393, "\u0120lacking": 14394, "\u0120celebration": 14395, "ju": 14396, "\u0120silly": 14397, "Euro": 14398, "\u0120tatt": 14399, "ighty": 14400, "controlled": 14401, "Test": 14402, "\u0120Singh": 14403, "\u0120rage": 14404, "\u0120rhyth": 14405, "offic": 14406, "\u0120Phantom": 14407, "\u0120headlines": 14408, "\u0120responding": 14409, "\u0120Morning": 14410, "\u0120vitamin": 14411, "\u0120boots": 14412, "\u0120Site": 14413, "alin": 14414, "pi": 14415, "\u0120viral": 14416, "\u0120UC": 14417, "DER": 14418, "\u0120Sex": 14419, "\u0120stocks": 14420, "current": 14421, "\u0120churches": 14422, "\u0120Rare": 14423, "\u0120Murphy": 14424, "\u0120denial": 14425, "\u0120Gaming": 14426, "\u0120toug": 14427, "\u0120nick": 14428, "\u0120makers": 14429, "\u0120Ronald": 14430, "\u0120generous": 14431, "\u0120Doc": 14432, "\u0120Morris": 14433, "\u0120transformed": 14434, "\u0120Normal": 14435, "\u0120104": 14436, "\u0120Kickstarter": 14437, "\u0120Upon": 14438, "Online": 14439, "\u0120IRS": 14440, "\u0120wrap": 14441, "\u0120loving": 14442, "\u0120arrives": 14443, "\u0120Due": 14444, "\u0120heter": 14445, "\u0120Made": 14446, "\u0120rental": 14447, "\u0120belongs": 14448, "\u0120attorneys": 14449, "\u0120crops": 14450, "\u0120matched": 14451, "ulum": 14452, "oline": 14453, "109": 14454, "\u0120dispar": 14455, "\u0120buyers": 14456, "\u0120Cambridge": 14457, "\u0120ethics": 14458, "roups": 14459, "\u0120justified": 14460, "\u0120marginal": 14461, "\u0120respected": 14462, "winning": 14463, "\u0120nodded": 14464, "\u0120Serge": 14465, "\u0120Former": 14466, "Craft": 14467, "################": 14468, "\u0120Warner": 14469, "\u0120dash": 14470, "ete": 14471, "\u0120entert": 14472, "\u0120Escape": 14473, "outheast": 14474, "\u0120knees": 14475, "\u0120Bomb": 14476, "\u0120rug": 14477, "Pass": 14478, "\u0120attitudes": 14479, "government": 14480, "\u0120Prior": 14481, "\u0120qualities": 14482, "\u0120notification": 14483, "\u0120Phone": 14484, "lie": 14485, "\u0120anticipated": 14486, "\u0120Combat": 14487, "\u0120Barry": 14488, "\u01201982": 14489, "Users": 14490, "oner": 14491, "\u0120computing": 14492, "\u0120Connecticut": 14493, "\u0120lesser": 14494, "\u0120peers": 14495, "\u0120Cu": 14496, "\u0120technically": 14497, "\u0120submission": 14498, "\u0120Universal": 14499, "\u0120manually": 14500, "ourge": 14501, "\u0120respondents": 14502, "\u0120BTC": 14503, "\u0120Host": 14504, "\u0120fare": 14505, "\u0120Bird": 14506, "\u0120receipt": 14507, "also": 14508, "\u0120jack": 14509, "\u0120agriculture": 14510, "\u0120skull": 14511, "\u0120!=": 14512, "\u0120passive": 14513, "\u0120CI": 14514, "\u0120societies": 14515, "\u0120reminded": 14516, "\u0120interference": 14517, "Buy": 14518, "\u0120\u00e2\u013e": 14519, "gon": 14520, "\u0120scrutiny": 14521, "\u0120Witch": 14522, "\u0120conducting": 14523, "\u0120\u00e3\u0125": 14524, "\u0120exchanges": 14525, "\u0120Mitchell": 14526, "\u0120inhabit": 14527, "\u0120twist": 14528, "BD": 14529, "\u0120wherever": 14530, "groupon": 14531, "\u0120jokes": 14532, "\u0120Benjamin": 14533, "\u0120Random": 14534, "frame": 14535, "\u0120Lions": 14536, "\u0120highlighted": 14537, "\u0120Arkansas": 14538, "Ent": 14539, "\u0120pile": 14540, "\u0120prelim": 14541, "gs": 14542, "minded": 14543, "\u0120felony": 14544, "\u0120GA": 14545, "\u0120Luck": 14546, "\u0120practically": 14547, "\u0120Bos": 14548, "\u0120actress": 14549, "Dam": 14550, "\u0120Bou": 14551, "\u0120visa": 14552, "\u0120embedded": 14553, "\u0120hybrid": 14554, "\u0120earliest": 14555, "\u0120sooner": 14556, "social": 14557, "\u0120HA": 14558, "\u0120steep": 14559, "\u0120disadvant": 14560, "\u0120exploit": 14561, "\u0120Egg": 14562, "\u0120Ultra": 14563, "\u0120necessity": 14564, "Local": 14565, "iege": 14566, "\u0120dated": 14567, "\u0120masses": 14568, "\u0120subscription": 14569, "pless": 14570, "\u0120anonym": 14571, "\u0120presumably": 14572, "Blue": 14573, "Their": 14574, "asketball": 14575, "\u0120Philip": 14576, "\u0120comed": 14577, "loaded": 14578, "rane": 14579, "\u0120reflection": 14580, "China": 14581, "\u0120extends": 14582, "\u0120forming": 14583, "\u0120unders": 14584, "2001": 14585, "\u0120grat": 14586, "\u0120concentrations": 14587, "\u0120insulin": 14588, "\u0120secular": 14589, "\u0120whilst": 14590, "\u0120winners": 14591, "Advertisements": 14592, "\u0120deliberately": 14593, "\u0120Working": 14594, "\u0120sink": 14595, "etics": 14596, "dale": 14597, "\u0120mandate": 14598, "\u0120gram": 14599, "\u0120vacation": 14600, "\u0120warnings": 14601, "ripp": 14602, "\u0120THAT": 14603, "\u0120commentary": 14604, "\u0120intu": 14605, "\u0120aest": 14606, "\u0120reasoning": 14607, "\u0120breakdown": 14608, "\u0120Zombie": 14609, "\u0120-->": 14610, "\u0120Political": 14611, "cott": 14612, "\u0120thrust": 14613, "\u0120technological": 14614, "\u0120deciding": 14615, "\u0120trafficking": 14616, "Long": 14617, "Welcome": 14618, "prising": 14619, "\u0120Communications": 14620, "\u0120endors": 14621, "\u0120swift": 14622, "\u0120metabol": 14623, "coins": 14624, "resa": 14625, "\u0120HTTP": 14626, "\u0120enroll": 14627, "\u0120Happy": 14628, "usr": 14629, "intage": 14630, "\u0120[\"": 14631, "uably": 14632, "\u0120Material": 14633, "\u0120repeal": 14634, "Sept": 14635, "kh": 14636, "\u0120Modi": 14637, "\u0120underneath": 14638, "\u0120IL": 14639, "shore": 14640, "\u0120diagnosed": 14641, "aceutical": 14642, "\u0120shower": 14643, "aux": 14644, "\u0120Switch": 14645, "\u0120Strength": 14646, "\u0120jihad": 14647, "national": 14648, "\u0120trauma": 14649, "ussy": 14650, "oni": 14651, "\u0120consolid": 14652, "\u0120calories": 14653, "\u0120Flynn": 14654, "agged": 14655, "168": 14656, "\u0120Pink": 14657, "\u0120fulfill": 14658, "\u0120chains": 14659, "\u0120notably": 14660, "\u0120AV": 14661, "Life": 14662, "\u0120Chuck": 14663, "mus": 14664, "\u0120Urban": 14665, "\u0120Hend": 14666, "\u0120deposit": 14667, "\u0120Sad": 14668, "\u0120affair": 14669, "ORK": 14670, "ieval": 14671, "\u0120FDA": 14672, "\u0120trop": 14673, "\u0120Overall": 14674, "\u0120virtue": 14675, "\u0120satisfaction": 14676, "aund": 14677, "\u0120lun": 14678, "\u0120Switzerland": 14679, "\u0120Operation": 14680, "process": 14681, "\u0120shook": 14682, "\u0120counties": 14683, "leased": 14684, "\u0120Charlotte": 14685, "112": 14686, "\u0120transcript": 14687, "\u0120redd": 14688, "push": 14689, "\u0120Hey": 14690, "\u0120Analysis": 14691, "[\"": 14692, "\u0120alternatives": 14693, "ardless": 14694, "\u0120eleph": 14695, "\u0120prejud": 14696, "\u0120Leaf": 14697, "Having": 14698, "\u0120Hub": 14699, "\u0120expressions": 14700, "\u0120Volume": 14701, "\u0120shocking": 14702, "\u0120Reds": 14703, "\u0120readily": 14704, "\u0120planets": 14705, "adata": 14706, "\u0120collapsed": 14707, "\u0120Madrid": 14708, "\u0120irrit": 14709, "ipper": 14710, "\u0120Enc": 14711, "\u0120Wire": 14712, "\u0120buzz": 14713, "\u0120GP": 14714, "asha": 14715, "\u0120accidentally": 14716, "uru": 14717, "\u0120frustrated": 14718, "\u0120SA": 14719, "\u0120hungry": 14720, "\u0120Huff": 14721, "\u0120labels": 14722, "anto": 14723, "\u0120EP": 14724, "\u0120barriers": 14725, ")|": 14726, "\u0120Berkeley": 14727, "\u0120Jets": 14728, "\u0120pairs": 14729, "\u0120Lan": 14730, "James": 14731, "\u0120Bear": 14732, "\u0120humor": 14733, "\u0120Liberty": 14734, "\u0120magnitude": 14735, "\u0120aging": 14736, "\u0120Mason": 14737, "\u0120friendship": 14738, "umbling": 14739, "\u0120emerge": 14740, "\u0120newspapers": 14741, "\u0120ambitious": 14742, "\u0120Richards": 14743, "aternal": 14744, "\u01201981": 14745, "\u0120cookies": 14746, "\u0120sculpt": 14747, "\u0120pursuit": 14748, "Location": 14749, "\u0120scripts": 14750, "pc": 14751, "\u0120arrangements": 14752, "\u0120diameter": 14753, "\u0120loses": 14754, "amation": 14755, "\u0120liqu": 14756, "\u0120Jake": 14757, "arette": 14758, "\u0120understands": 14759, "\u0120Zen": 14760, "vm": 14761, "\u0120approve": 14762, "\u0120wip": 14763, "\u0120ultra": 14764, "\u0120intend": 14765, "\u0120DI": 14766, "ascular": 14767, "\u0120stays": 14768, "\u0120Kor": 14769, "\u0120Kl": 14770, "\u0120investing": 14771, "La": 14772, "\u0120believing": 14773, "bad": 14774, "mouth": 14775, "\u0120taxpayer": 14776, "\u00e3\u0125\u0125": 14777, "\u0120Quebec": 14778, "\u0120lap": 14779, "\u0120Swiss": 14780, "drop": 14781, "\u0120drain": 14782, "iri": 14783, "etc": 14784, "ften": 14785, "\u0120Nex": 14786, "\u0120straw": 14787, "\u0120screaming": 14788, "\u0120counted": 14789, "\u0120damaging": 14790, "\u0120ambassador": 14791, "century": 14792, "\u0120prox": 14793, "\u0120arrests": 14794, "uv": 14795, "ilateral": 14796, "\u0120Charg": 14797, "\u0120prescribed": 14798, "\u0120independently": 14799, "\u0120fierce": 14800, "\u0120Baby": 14801, "\u0120brave": 14802, "\u0120suits": 14803, "=>": 14804, "\u0120baseline": 14805, "\u0120Rate": 14806, "\u0120islands": 14807, "\u0120((": 14808, "green": 14809, "ixels": 14810, "\u0120namely": 14811, "\u0120Village": 14812, "than": 14813, "amy": 14814, "Version": 14815, "gmail": 14816, "entials": 14817, "\u0120Sud": 14818, "\u0120Melbourne": 14819, "\u0120arriving": 14820, "\u0120quantum": 14821, "eff": 14822, "ropolitan": 14823, "Tri": 14824, "\u0120funeral": 14825, "\u0120IR": 14826, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 14827, "\u0120Cob": 14828, "itably": 14829, "\u0120turb": 14830, "\u0120combo": 14831, "Review": 14832, "\u0120deployment": 14833, "uity": 14834, "\u0120Bott": 14835, "\u0120invisible": 14836, "\u0120rendering": 14837, "\u0120unlocked": 14838, "\u0120aqu": 14839, "\u0120Vladimir": 14840, "\u0120pad": 14841, "\u0120Brain": 14842, "\u0120Legacy": 14843, "dragon": 14844, "\u0120Kurdish": 14845, "\u0120sounded": 14846, "\u0120detained": 14847, "\u0120DM": 14848, "gary": 14849, "\u0120daughters": 14850, "\u0120disturbing": 14851, "uka": 14852, "\u0120Parad": 14853, "\u0120tast": 14854, "\u0120unfortunate": 14855, "\u0120ul": 14856, "emin": 14857, "\u0120attendance": 14858, "trl": 14859, "\u0120parks": 14860, "\u0120Memorial": 14861, "\u0120Alice": 14862, "othy": 14863, "guard": 14864, "\u0120Dise": 14865, "\u0120Shan": 14866, "\u0120Forum": 14867, "Rich": 14868, "\u0120shifted": 14869, "uez": 14870, "\u0120lighter": 14871, "\u0120Magn": 14872, "\u0120cod": 14873, "Sch": 14874, "hammad": 14875, "Pub": 14876, "350": 14877, "\u0120Pokemon": 14878, "\u0120prototype": 14879, "\u0120unre": 14880, "Base": 14881, "\u0120Students": 14882, "\u0120Reply": 14883, "\u0120Communist": 14884, "\u0120gau": 14885, "\u0120Tyler": 14886, "IZ": 14887, "\u0120participated": 14888, "\u0120suprem": 14889, "\u0120Details": 14890, "\u0120vessels": 14891, "rod": 14892, "\u0120tribe": 14893, "keep": 14894, "\u0120assumptions": 14895, "\u0120pound": 14896, "\u0120crude": 14897, "\u0120Available": 14898, "\u0120swimming": 14899, "\u0120inclusion": 14900, "\u0120advances": 14901, "culation": 14902, "\u0120conservation": 14903, "\u0120overd": 14904, "\u0120Buffalo": 14905, "Article": 14906, "edge": 14907, "\u0120awa": 14908, "\u0120Madison": 14909, "\u0120sidew": 14910, "\u0120catast": 14911, "\u0120Krist": 14912, "ucle": 14913, "\u0120Highway": 14914, "\u0120Terror": 14915, "\u0120activation": 14916, "\u0120unconscious": 14917, "\u0120Satan": 14918, "\u0120Susan": 14919, "illery": 14920, "\u0120arranged": 14921, "iop": 14922, "\u0120rumors": 14923, "urring": 14924, "think": 14925, "\u0120Keith": 14926, "\u0120Kind": 14927, "\u0120avoiding": 14928, "byn": 14929, "nut": 14930, "\u0120Speaker": 14931, "rus": 14932, "names": 14933, "\u0120guilt": 14934, "\u0120Olympics": 14935, "\u0120sail": 14936, "\u0120Mes": 14937, "levant": 14938, "\u0120Columbus": 14939, "aft": 14940, "City": 14941, "South": 14942, "\u0120Harvey": 14943, "\u0120Pun": 14944, "Several": 14945, "\u0120mentally": 14946, "\u0120impress": 14947, "mount": 14948, "\u0120Ubuntu": 14949, "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 14950, "\u0120Superman": 14951, "\u0120MPs": 14952, "\u0120intentions": 14953, "\u0120Racing": 14954, "\u0120likelihood": 14955, "\u0120240": 14956, "Total": 14957, "\u0120toys": 14958, "\u0120Watson": 14959, "\u0120urge": 14960, "Lear": 14961, "\u0120Paper": 14962, "\u0120occurring": 14963, "\u0120Beng": 14964, "\u0120Cert": 14965, "\u0120stones": 14966, "Tim": 14967, "\u0120Twin": 14968, "zb": 14969, "\u0120Dynam": 14970, "\u0120politician": 14971, "kens": 14972, "\u0120Enterprise": 14973, "UTERS": 14974, "\u0120abol": 14975, "\u0120refresh": 14976, "\u0120arbitrary": 14977, "pection": 14978, "\u0120troubles": 14979, "\u0120});": 14980, "tv": 14981, "\u0120pilots": 14982, "\u0120distribute": 14983, "\u0120audit": 14984, "\u0120pause": 14985, "original": 14986, "\u0120rivals": 14987, "\u00c2\u00a3": 14988, "Fig": 14989, "TL": 14990, "abil": 14991, "rying": 14992, "Lin": 14993, "ioned": 14994, "lon": 14995, "\u0120fancy": 14996, "\u0120crashed": 14997, "\u0120tract": 14998, "\u0120shed": 14999, "\u0120consume": 15000, "Based": 15001, "download": 15002, "init": 15003, "\u0120voltage": 15004, "Introdu": 15005, "\u0120condemned": 15006, "\u0120Finance": 15007, "respect": 15008, "\u0120excluded": 15009, "\u0120establishing": 15010, "heric": 15011, "\u0120heritage": 15012, "\u0120spectacular": 15013, "\u0120unst": 15014, "\u0120Snowden": 15015, "\u0120Lane": 15016, "San": 15017, "\u0120protections": 15018, "struction": 15019, "incinn": 15020, "\u0120macro": 15021, "Custom": 15022, "iosity": 15023, "\u0120esp": 15024, "\u0120functioning": 15025, "\u0120mush": 15026, "\u0120puzzle": 15027, "\u0120ethical": 15028, "Mal": 15029, "\u0120governing": 15030, "\u0120Ferguson": 15031, "\u0120restored": 15032, "\u0120stressed": 15033, "\u0120Counter": 15034, "\u0120Kas": 15035, "clip": 15036, "ANS": 15037, "\u0120seiz": 15038, "UK": 15039, "byss": 15040, "oldown": 15041, "api": 15042, "\u0120permanently": 15043, "ounters": 15044, "West": 15045, "Through": 15046, "Light": 15047, "atoes": 15048, "\u0120neat": 15049, "\u0120cord": 15050, "urer": 15051, "\u0120severely": 15052, "\u0120Aven": 15053, "\u0120interrog": 15054, "\u0120triple": 15055, "Given": 15056, "Number": 15057, "\u0120arise": 15058, "\u0120sher": 15059, "plant": 15060, "\u0120flower": 15061, "\u0120Cou": 15062, "\u0120ate": 15063, "\u0120newer": 15064, "bul": 15065, "\u0120meanwhile": 15066, "\u0120Lair": 15067, "\u0120adjustment": 15068, "\u0120Copyright": 15069, "\u0120divers": 15070, "iological": 15071, "\u0120gamers": 15072, "oat": 15073, "\u0120historically": 15074, "\u0120analog": 15075, "\u0120longtime": 15076, "\u0120prescription": 15077, "\u0120Mist": 15078, "\u0120Hyper": 15079, "\u0120Maine": 15080, "\u0120Deity": 15081, "\u0120multipl": 15082, "\u0120Reincarn": 15083, "\u0120Hyd": 15084, "\u0120Pic": 15085, "Sil": 15086, "rants": 15087, "\u0120Cris": 15088, ".;": 15089, "({": 15090, "ependence": 15091, "\u0120recy": 15092, "ateur": 15093, "\u0120quad": 15094, "\u0120glob": 15095, "\u0120conced": 15096, "team": 15097, "\u0120capitalist": 15098, "\u0120Lot": 15099, "\u0120royal": 15100, "\u0120Cyber": 15101, "\u0120blacks": 15102, "metic": 15103, "riv": 15104, "\u0120Danny": 15105, "\u0120spo": 15106, "\u0120RO": 15107, "\u0120animated": 15108, "rypted": 15109, "\u0120Deputy": 15110, "\u0120rendered": 15111, "FE": 15112, "\u0120streak": 15113, "\u0120clouds": 15114, "\u0120Doug": 15115, "~~~~~~~~": 15116, "\u0120discour": 15117, "\u0120Veh": 15118, "\u0120psychology": 15119, "\u0120Journey": 15120, "\u0120crystal": 15121, "\u0120Frost": 15122, "\u0120suspicion": 15123, "\u0120relate": 15124, "orus": 15125, "\u0120Crypt": 15126, "\u0120NVIDIA": 15127, "comed": 15128, "uting": 15129, "incinnati": 15130, "\u0120vulnerability": 15131, "ostic": 15132, "\u0120isolation": 15133, "\u0120cooling": 15134, "\u0120Coalition": 15135, "\u0120119": 15136, "Four": 15137, "\u0120Deal": 15138, "\u0120\u00e2\u012b": 15139, "semble": 15140, "rament": 15141, "\u0120Barcelona": 15142, "\u0120102": 15143, "\u0120cocaine": 15144, "ocalypse": 15145, "Feb": 15146, "ogenic": 15147, "\u0120mutation": 15148, "\u0120cryptoc": 15149, "\u0120Kel": 15150, "\u0120Git": 15151, "ais": 15152, "\u0120sisters": 15153, "ANK": 15154, "\u0120activate": 15155, "Ter": 15156, "\u0120dread": 15157, "ylon": 15158, "\u0120propri": 15159, "Aust": 15160, "\u0120Default": 15161, "\u0120outdoor": 15162, "\u0120sheer": 15163, "ceive": 15164, "\u0120gently": 15165, "\u00d0\u00be": 15166, "Program": 15167, "\u0120\u00e2\u0128\u0134": 15168, "\u0120vegan": 15169, "\u0120Crus": 15170, "\u0120responsibilities": 15171, "\u0120HR": 15172, "OLD": 15173, "\u0120prevents": 15174, "\u0120stiff": 15175, "\u0120Were": 15176, "\u0120athletic": 15177, "\u0120Score": 15178, "\u0120):": 15179, "\u0120columns": 15180, "\u0120Loc": 15181, "available": 15182, "\u0120Fram": 15183, "\u0120Sessions": 15184, "\u0120companion": 15185, "\u0120packs": 15186, "140": 15187, "\u0120Knights": 15188, "\u0120fart": 15189, "\u0120streams": 15190, "\u0120shore": 15191, "\u0120appeals": 15192, "\u0120Performance": 15193, "haul": 15194, "\u0120Stra": 15195, "\u0120Nag": 15196, "103": 15197, "\u0120Transportation": 15198, "BB": 15199, "Ev": 15200, "zan": 15201, "Public": 15202, "\u0120twin": 15203, "ulsion": 15204, "Mult": 15205, "\u0120electro": 15206, "\u0120statue": 15207, "ationally": 15208, "\u0120Nort": 15209, "\u0120inspection": 15210, "/*": 15211, "igue": 15212, "\u0120compassion": 15213, "\u0120Tales": 15214, "\u0120Stein": 15215, "\u0120Screen": 15216, "\u0120Bug": 15217, "\u0120Lion": 15218, "girl": 15219, "\u0120withdrawal": 15220, "\u0120objectives": 15221, "\u0120bloody": 15222, "\u0120preliminary": 15223, "\u0120jacket": 15224, "\u0120dimensions": 15225, "\u0120Cool": 15226, "\u0120Occup": 15227, "\u0120wreck": 15228, "\u0120doubled": 15229, "anking": 15230, "\u01201975": 15231, "\u0120glasses": 15232, "\u0120Wang": 15233, "prov": 15234, "Path": 15235, "connected": 15236, "\u0120Multi": 15237, "\u0120Norway": 15238, "agonist": 15239, "\u0120feared": 15240, "\u0120touching": 15241, "\u0120arguably": 15242, "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 15243, "\u0120NCAA": 15244, "chem": 15245, "\u0120spat": 15246, "\u0120WWE": 15247, "\u0120Cel": 15248, "igger": 15249, "\u0120attacker": 15250, "\u0120Join": 15251, "object": 15252, "etta": 15253, "\u0120eliminated": 15254, "det": 15255, "\u0120destruct": 15256, "\u0120Lucas": 15257, "ctuary": 15258, "180": 15259, "\u0120Brady": 15260, "\u0120Blues": 15261, "Bay": 15262, "aukee": 15263, "\u0120timeline": 15264, "\u0120delegates": 15265, "written": 15266, "ufficient": 15267, "\u0120shapes": 15268, "Copyright": 15269, "ouble": 15270, "service": 15271, "\u0120pione": 15272, "\u0120colleges": 15273, "\u0120rows": 15274, "\u0120spite": 15275, "\u0120assessed": 15276, "360": 15277, "\u0120lease": 15278, "\u0120confidential": 15279, "cker": 15280, "\u0120Manning": 15281, "\u0120Voice": 15282, "\u0120sealed": 15283, "\u0120calculate": 15284, "NO": 15285, "\u0120Assistant": 15286, "\u0120teenager": 15287, "ulent": 15288, "atherine": 15289, "\u0120mock": 15290, "\u0120diamond": 15291, "\u0120fest": 15292, "\u0120switched": 15293, "\u0120resume": 15294, "\u0120Puerto": 15295, "\u0120lanes": 15296, "iration": 15297, "\u0120Similarly": 15298, "\u0120rod": 15299, "\u0120Sel": 15300, "\u0120Palace": 15301, "\u0120Limited": 15302, "eous": 15303, "\u0120variant": 15304, "\u0120ward": 15305, "\u0120))": 15306, "Show": 15307, "OOK": 15308, "Alex": 15309, "\u0120Nep": 15310, "bris": 15311, "\u0120Wikipedia": 15312, "\u0120exceptional": 15313, "\u0120manages": 15314, "\u0120Draw": 15315, "Again": 15316, "\u0120copper": 15317, "utt": 15318, "\u0120exports": 15319, "\u0120portfolio": 15320, "\u0120elevated": 15321, "Rated": 15322, "\u0120Otherwise": 15323, "\u0120Tact": 15324, "\u0120Shel": 15325, "\u0120TX": 15326, "\"\u00e2\u0122\u0136": 15327, "\u0120resur": 15328, "\u0120Wa": 15329, "venant": 15330, "\u0120monetary": 15331, "people": 15332, "Email": 15333, "\u0120fifty": 15334, "\u0120Sweet": 15335, "\u0120Malaysia": 15336, "\u0120confusing": 15337, "\u0120Rio": 15338, "uda": 15339, "utenant": 15340, "\");": 15341, "\u0120praised": 15342, "\u0120volumes": 15343, "turn": 15344, "\u0120mature": 15345, "\u0120nonprofit": 15346, "\u0120passionate": 15347, "\u0120Private": 15348, "\u0120103": 15349, "\u0120descend": 15350, "\u00e7\u00a5\u0140": 15351, "uffy": 15352, "headed": 15353, "Whether": 15354, "rien": 15355, "zech": 15356, "beit": 15357, "\u0120chrom": 15358, "\u0120McM": 15359, "\u0120dancing": 15360, "\u0120eleg": 15361, "\u0120Noticed": 15362, "115": 15363, "\u0120advocacy": 15364, "ENTS": 15365, "ambling": 15366, "\u0120Minor": 15367, "\u0120Finn": 15368, "\u0120priorities": 15369, "\u0120thereof": 15370, "\u0120Stage": 15371, "\u0120Rogers": 15372, "\u0120substitute": 15373, "\u0120Jar": 15374, "\u0120Jefferson": 15375, "\u0120lightly": 15376, "102": 15377, "\u0120Lisa": 15378, "uits": 15379, "ysical": 15380, "\u0120shifts": 15381, "\u0120drones": 15382, "\u0120workplace": 15383, "\u0120resid": 15384, "ensed": 15385, "ahn": 15386, "\u0120preferences": 15387, "server": 15388, "\u0120debates": 15389, "doc": 15390, "\u0120Gods": 15391, "\u0120helicopter": 15392, "\u0120honour": 15393, "\u0120considerably": 15394, "eded": 15395, "\u0120Female": 15396, "\u0120Anne": 15397, "\u0120reun": 15398, "\u0120Face": 15399, "\u0120Hallow": 15400, "\u0120Budget": 15401, "\u0120condemn": 15402, "\u0120tender": 15403, "Prof": 15404, "ocratic": 15405, "\u0120Turner": 15406, "\u0120Agric": 15407, "\u01201976": 15408, "\u0120apt": 15409, "disc": 15410, "\u0120Fighter": 15411, "\u0120Aur": 15412, "\u0120garbage": 15413, "input": 15414, "\u0120Karl": 15415, "\u0120Oliver": 15416, "\u0120Language": 15417, "kn": 15418, "Non": 15419, "\u0120Clar": 15420, "\u0120traditions": 15421, "\u0120advertisement": 15422, "\u0120Sor": 15423, "\u0120archive": 15424, "\u0120villages": 15425, "750": 15426, "\u0120implementing": 15427, "waukee": 15428, "\u0120dietary": 15429, "\u0120switching": 15430, "Republic": 15431, "\u0120velocity": 15432, "\u0120cit": 15433, "\u0120Awards": 15434, "\u0120financing": 15435, "\u0120lasted": 15436, ")]": 15437, "\u0120reminder": 15438, "Person": 15439, "\u0120precision": 15440, "\u0120designers": 15441, "\u0120Fried": 15442, "\u0120Border": 15443, "\u0120tragic": 15444, "\u0120wield": 15445, "\u0120initiatives": 15446, "\u0120Tank": 15447, "wer": 15448, "\u0120joins": 15449, "Ro": 15450, "inery": 15451, "\u0120arrow": 15452, "\u0120generating": 15453, "founder": 15454, "\u0120searches": 15455, "\u0120randomly": 15456, "Access": 15457, "\u0120batch": 15458, "\u0120posed": 15459, "lat": 15460, "\u0120pursuing": 15461, "asa": 15462, "\u0120testified": 15463, "forming": 15464, "\u0120Shar": 15465, "wiki": 15466, "\u0120Either": 15467, "Sometimes": 15468, "\u0120senators": 15469, "\u0120Johnny": 15470, "\u0120Taliban": 15471, "\u0120GPS": 15472, "\":\"/": 15473, "\u00e3\u0123\u00ae\u00e5": 15474, "\u0120analyzed": 15475, "\u0120Rubio": 15476, "\u0120Movement": 15477, "opard": 15478, "iii": 15479, "Stand": 15480, "fight": 15481, "\u0120ignoring": 15482, "iang": 15483, "\u0120GN": 15484, "soever": 15485, "\u0120STAT": 15486, "\u0120refusing": 15487, "\u0120sweat": 15488, "\u0120bay": 15489, "PORT": 15490, "irmed": 15491, "aky": 15492, "\u0120dispro": 15493, "\u0120labeled": 15494, "\u0120108": 15495, "Hello": 15496, "\u0120pleasant": 15497, "aba": 15498, "\u0120triumph": 15499, "\u0120aboard": 15500, "\u0120incom": 15501, "\u0120Crow": 15502, "lett": 15503, "\u0120folk": 15504, "\u0120chase": 15505, "``": 15506, "\u0120Brus": 15507, "\u0120teens": 15508, "cue": 15509, "\u0120terrain": 15510, "hyd": 15511, "ilight": 15512, "ORY": 15513, "Support": 15514, "ews": 15515, "lli": 15516, "raints": 15517, "\u0120Cand": 15518, "\u0120abused": 15519, "achment": 15520, "larg": 15521, "Bas": 15522, "\u0120Cancer": 15523, "\u01201978": 15524, "\u0120supporter": 15525, "access": 15526, "\u0120Termin": 15527, "\u0120Tampa": 15528, "\u0120ANY": 15529, "\u0120newest": 15530, "\u0120Criminal": 15531, "edu": 15532, "\u01201930": 15533, "\u0120admits": 15534, "\u0120ende": 15535, "\u0120failures": 15536, "urate": 15537, "fulness": 15538, "cycl": 15539, "\u0120Subject": 15540, "\u0120infinite": 15541, "three": 15542, "WA": 15543, "pit": 15544, "\u0120Install": 15545, "Rad": 15546, "iliation": 15547, "GM": 15548, "\u0120continent": 15549, "\u0120accommodate": 15550, "\u0120Clay": 15551, "\u0120pup": 15552, "\u0120Function": 15553, "\u0120hammer": 15554, "\u0120Alberta": 15555, "\u0120revised": 15556, "\u0120minorities": 15557, "\u0120measurement": 15558, "Connell": 15559, "\u0120disable": 15560, "\u0120Mix": 15561, "Incre": 15562, "\u0120fork": 15563, "\u0120Rosen": 15564, "\u0120implies": 15565, "umblr": 15566, "ANG": 15567, "\u0120proteins": 15568, "\u0120aggression": 15569, "\u0120facilitate": 15570, "SN": 15571, "\u0120illegally": 15572, "uer": 15573, "\u0120academ": 15574, "\u0120puzz": 15575, "\u0120Shift": 15576, "pay": 15577, "ollo": 15578, "\u0120audiences": 15579, "Build": 15580, "\u0120noble": 15581, "\u0120syntax": 15582, "\u00e2\u013a\u0127": 15583, "\u0120beam": 15584, "\u0120Bed": 15585, "\u0120Ald": 15586, "\u0120origins": 15587, "video": 15588, "\u01201977": 15589, "\u0120Assault": 15590, "\u0120garage": 15591, "Team": 15592, "\u0120verdict": 15593, "\u0120dwar": 15594, "\u0120Virtual": 15595, "event": 15596, "Keep": 15597, "\u0120sentiment": 15598, "\u0120wildlife": 15599, "shirt": 15600, "\u0120burg": 15601, "\u0120recommendation": 15602, "represent": 15603, "\u0120gallery": 15604, "owners": 15605, "\u0120scholar": 15606, "\u0120convenience": 15607, "\u0120Swift": 15608, "\u0120convinc": 15609, "Cap": 15610, "\u0120warfare": 15611, "\u0120Visual": 15612, "\u0120constitute": 15613, "\u0120abort": 15614, "\u0120Weather": 15615, "\u0120Looking": 15616, "\u0120Hem": 15617, "\u0120martial": 15618, "\u0120incoming": 15619, "etition": 15620, "\u0120tolerance": 15621, "\u0120Created": 15622, "\u0120flows": 15623, "\u0120Elder": 15624, "\u0120souls": 15625, "\u0120foul": 15626, "\u0120Pain": 15627, "\u0120CAN": 15628, "\u0120220": 15629, "bc": 15630, "hend": 15631, "\u0120genius": 15632, "Real": 15633, "\u0120Wr": 15634, "ometer": 15635, "pad": 15636, "\u0120limiting": 15637, "\u0120Si": 15638, "\u0120Lore": 15639, "\u0120Adventures": 15640, "\u0120varied": 15641, "Disc": 15642, "fin": 15643, "\u0120Personal": 15644, "Chris": 15645, "\u0120invented": 15646, "\u0120dive": 15647, "\u0120Rise": 15648, "\u0120oz": 15649, "\u0120Comics": 15650, "\u0120expose": 15651, "\u0120Reb": 15652, "letters": 15653, "site": 15654, "imated": 15655, "\u0120hacking": 15656, "\u0120educated": 15657, "\u0120Nobody": 15658, "\u0120depri": 15659, "\u0120incentive": 15660, "\u00e3\u0124\u00b7": 15661, "\u0120oversight": 15662, "\u0120tribes": 15663, "\u0120Belgium": 15664, "\u0120licensing": 15665, "ourt": 15666, "Product": 15667, "ahl": 15668, "\u0120Gem": 15669, "\u0120specialist": 15670, "\u0120cra": 15671, "anners": 15672, "\u0120Corbyn": 15673, "\u01201973": 15674, "READ": 15675, "\u0120summar": 15676, "\u0120overlook": 15677, "\u0120Application": 15678, "\u0120inappropriate": 15679, "\u0120downloaded": 15680, "Que": 15681, "\u0120Bears": 15682, "\u0120thumb": 15683, "\u0120Character": 15684, "\u0120Reincarnated": 15685, "\u0120Sid": 15686, "\u0120demonstrates": 15687, "sky": 15688, "\u0120Bloomberg": 15689, "\u0120Array": 15690, "\u0120Results": 15691, "\u0120Fourth": 15692, "\u0120EDT": 15693, "\u0120Oscar": 15694, "cend": 15695, "\u0120106": 15696, "\u0120NULL": 15697, "\u0120HERE": 15698, "match": 15699, "\u0120Brun": 15700, "\u0120glucose": 15701, "ieg": 15702, "egu": 15703, "\u0120certified": 15704, "\u0120relie": 15705, "\u0120humanitarian": 15706, "\u0120prayers": 15707, "King": 15708, "\u0120nan": 15709, "hou": 15710, "108": 15711, "ulu": 15712, "\u0120renewable": 15713, "\u0120distinguish": 15714, "\u0120dense": 15715, "\u0120Vent": 15716, "\u0120Package": 15717, "\u0120Boss": 15718, "\u0120editors": 15719, "\u0120migr": 15720, "Tra": 15721, "\u0120Peters": 15722, "\u0120Arctic": 15723, "2004": 15724, "\u0120Cape": 15725, "\u0120locally": 15726, "\u0120lasting": 15727, "\u0120handy": 15728, ".).": 15729, "Pan": 15730, "\u0120RES": 15731, "Index": 15732, "\u0120tensions": 15733, "\u0120formerly": 15734, "\u0120ideological": 15735, "\u0120sensors": 15736, "\u0120dealers": 15737, "\u0120defines": 15738, "Sk": 15739, "\u0120proceeds": 15740, "\u0120proxy": 15741, "azines": 15742, "\u0120Bash": 15743, "\u0120Pad": 15744, "\u0120Craft": 15745, "ealous": 15746, "\u0120sheets": 15747, "ometry": 15748, "June": 15749, "clock": 15750, "TT": 15751, "\u0120Theatre": 15752, "\u0120Buzz": 15753, "\u0120chapters": 15754, "\u0120millenn": 15755, "\u0120dough": 15756, "\u0120Congressional": 15757, "\u0120imagined": 15758, "avior": 15759, "\u0120clinic": 15760, "\u01201945": 15761, "\u0120holder": 15762, "root": 15763, "olester": 15764, "\u0120restart": 15765, "BN": 15766, "\u0120Hamas": 15767, "\u0120Job": 15768, "\u0120orb": 15769, "\u0120ram": 15770, "\u0120disclose": 15771, "\u0120translate": 15772, "\u0120immigrant": 15773, "\u0120annoying": 15774, "\u0120treaty": 15775, "anium": 15776, "\u0120Tea": 15777, "\u0120Legion": 15778, "\u0120crowds": 15779, "\u0120Bec": 15780, "\u0120Aer": 15781, "ohyd": 15782, "Bro": 15783, "Looking": 15784, "\u0120lbs": 15785, "\u0120aggress": 15786, "\u0120seam": 15787, "\u0120intercept": 15788, "\u0120MI": 15789, "mercial": 15790, "activ": 15791, "\u0120Cit": 15792, "\u0120dimension": 15793, "\u0120consistency": 15794, "\u0120rushing": 15795, "\u0120Douglas": 15796, "\u0120trim": 15797, "Install": 15798, "icker": 15799, "\u0120shy": 15800, "106": 15801, "\u0120mentions": 15802, "pelled": 15803, "\u0120Tak": 15804, "cost": 15805, "\u0120classroom": 15806, "\u0120fortune": 15807, "driven": 15808, "\u0120unle": 15809, "\u0120Wheel": 15810, "\u0120investor": 15811, "\u0120Masters": 15812, "kit": 15813, "\u0120associations": 15814, "\u0120Evolution": 15815, "oping": 15816, "uscript": 15817, "\u0120provincial": 15818, "\u0120Walter": 15819, "avi": 15820, "SO": 15821, "\u0120unlimited": 15822, "English": 15823, "\u0120Cards": 15824, "\u0120Ebola": 15825, "nered": 15826, "\u0120revenge": 15827, "\u0120outright": 15828, "umper": 15829, "\u0120fitting": 15830, "\u0120Solid": 15831, "\u0120formally": 15832, "\u0120problematic": 15833, "\u0120hazard": 15834, "\u0120encryption": 15835, "\u0120straightforward": 15836, "\u0120AK": 15837, "\u0120pse": 15838, "\u0120Orb": 15839, "\u0120Chamber": 15840, "\u0120Mak": 15841, "Contents": 15842, "\u0120loyalty": 15843, "\u0120lyrics": 15844, "\u0120Sym": 15845, "\u0120welcomed": 15846, "\u0120cooked": 15847, "\u0120monop": 15848, "\u0120nurse": 15849, "\u0120misleading": 15850, "\u0120eternal": 15851, "\u0120shifting": 15852, "\u0120+=": 15853, "Vis": 15854, "\u0120institutional": 15855, "illary": 15856, "\u0120pant": 15857, "VERT": 15858, "\u0120ACC": 15859, "\u0120Enh": 15860, "\u0120incon": 15861, "\u0120REUTERS": 15862, "\u0120donated": 15863, "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 15864, "Intern": 15865, "\u0120exhibit": 15866, "\u0120tire": 15867, "\u0120Ric": 15868, "\u0120Champion": 15869, "\u0120Muhammad": 15870, "NING": 15871, "\u0120Soccer": 15872, "\u0120mobility": 15873, "\u0120varying": 15874, "\u0120Movie": 15875, "\u0120lord": 15876, "oak": 15877, "Field": 15878, "\u0120vector": 15879, "usions": 15880, "\u0120scrap": 15881, "\u0120enabling": 15882, "make": 15883, "Tor": 15884, ".*": 15885, "||": 15886, "\u0120Website": 15887, "\u0120NPC": 15888, "\u0120socialist": 15889, "\u0120Billy": 15890, "\u0120Additional": 15891, "\u0120cargo": 15892, "\u0120farms": 15893, "\u0120Soon": 15894, "\u0120Prize": 15895, "\u0120midnight": 15896, "\u0120900": 15897, "seen": 15898, "\u0120Spot": 15899, "\u0120sheep": 15900, "\u0120sponsored": 15901, "\u0120Hi": 15902, "\u0120Jump": 15903, "\u01201967": 15904, "Microsoft": 15905, "\u0120Agent": 15906, "\u0120charts": 15907, "dir": 15908, "\u0120adjacent": 15909, "\u0120tricks": 15910, "\u0120manga": 15911, "\u0120exagger": 15912, "/>": 15913, "football": 15914, "\u0120FCC": 15915, "GC": 15916, "\u0120Tier": 15917, "andra": 15918, "OUND": 15919, "%),": 15920, "\u0120fruits": 15921, "VC": 15922, "\u0120AA": 15923, "Rober": 15924, "\u0120midst": 15925, "\u00e2\u0139": 15926, "anka": 15927, "\u0120legislature": 15928, "\u0120Neil": 15929, "\u0120tourists": 15930, "\"\"": 15931, "\u0120Warning": 15932, "\u0120Nevertheless": 15933, "\u0120Official": 15934, "\u0120Whatever": 15935, "\u0120mold": 15936, "\u0120drafted": 15937, "\u0120substances": 15938, "\u0120breed": 15939, "\u0120tags": 15940, "\u0120Task": 15941, "\u0120verb": 15942, "\u0120manufactured": 15943, "comments": 15944, "\u0120Polish": 15945, "Prov": 15946, "\u0120determines": 15947, "Obama": 15948, "kers": 15949, "\u0120utterly": 15950, "\u0120sect": 15951, "sche": 15952, "\u0120Gates": 15953, "\u0120Chap": 15954, "\u0120aluminum": 15955, "\u0120zombie": 15956, "\u0120Touch": 15957, "\u0120UP": 15958, "\u0120satisfy": 15959, "\u0120predomin": 15960, "ascript": 15961, "\u0120elaborate": 15962, "\u01201968": 15963, "\u0120measuring": 15964, "\u0120Vari": 15965, "anyahu": 15966, "\u0120sir": 15967, "ulates": 15968, "idges": 15969, "ickets": 15970, "\u0120Spencer": 15971, "TM": 15972, "oubted": 15973, "\u0120prey": 15974, "\u0120installing": 15975, "\u0120Cab": 15976, "reed": 15977, "reated": 15978, "Supp": 15979, "\u0120wrist": 15980, "\u0120Kerry": 15981, "107": 15982, "\u0120Kle": 15983, "\u0120Rachel": 15984, "\u0120cotton": 15985, "\u0120ARE": 15986, "\u0120Ele": 15987, "Control": 15988, "\u0120loads": 15989, "\u0120Dod": 15990, "anas": 15991, "bone": 15992, "\u0120classical": 15993, "\u0120Regional": 15994, "\u0120Integ": 15995, "VM": 15996, "\u0120desires": 15997, "\u0120autism": 15998, "supported": 15999, "\u0120Message": 16000, "\u0120compact": 16001, "writer": 16002, "\u0120109": 16003, "\u0120Hurricane": 16004, "cision": 16005, "\u0120cycles": 16006, "\u0120drill": 16007, "\u0120colleague": 16008, "\u0120maker": 16009, "German": 16010, "\u0120mistaken": 16011, "Sun": 16012, "\u0120Gay": 16013, "\u0120whatsoever": 16014, "\u0120sells": 16015, "\u0120Airl": 16016, "liv": 16017, "\u0120Option": 16018, "\u0120solved": 16019, "\u0120sectors": 16020, "\u0120horizontal": 16021, "\u0120equation": 16022, "\u0120Skill": 16023, "\u0120Bio": 16024, "gement": 16025, "\u0120Snap": 16026, "\u0120Legal": 16027, "\u0120trademark": 16028, "\u0120makeup": 16029, "\u0120assembled": 16030, "\u0120saves": 16031, "\u0120Halloween": 16032, "\u0120Vermont": 16033, "\u0120FROM": 16034, "\u0120farming": 16035, "\u0120Podcast": 16036, "acceptable": 16037, "\u0120Higher": 16038, "\u0120asleep": 16039, "ullivan": 16040, "\u0120referen": 16041, "\u0120Lev": 16042, "\u0120bullets": 16043, "oko": 16044, "HC": 16045, "\u0120stairs": 16046, "\u0120maintains": 16047, "\u0120Lower": 16048, "\u0120Vi": 16049, "\u0120marine": 16050, "\u0120acres": 16051, "\u0120coordinator": 16052, "\u0120Joh": 16053, "\u0120counterparts": 16054, "\u0120Brothers": 16055, "\u0120indict": 16056, "bra": 16057, "\u0120chunk": 16058, "\u0120cents": 16059, "Home": 16060, "\u0120Month": 16061, "\u0120accordingly": 16062, "ifles": 16063, "\u0120Germans": 16064, "\u0120Syn": 16065, "Hub": 16066, "\u0120eyeb": 16067, "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 16068, "\u0120ranges": 16069, "\u0120Holland": 16070, "\u0120Robot": 16071, "fc": 16072, "Mike": 16073, "\u0120plasma": 16074, "\u0120swap": 16075, "\u0120athlete": 16076, "\u0120Rams": 16077, ",'\"": 16078, "\u0120infections": 16079, "\u0120corrid": 16080, "\u0120vib": 16081, "\u0120patches": 16082, "\u0120traditionally": 16083, "\u0120revelation": 16084, "\u0120sweep": 16085, "\u0120glance": 16086, "\u0120inex": 16087, "2003": 16088, "\u0120Raw": 16089, "working": 16090, "osures": 16091, "\u0120Dat": 16092, "\u0120Lynch": 16093, "\u0120leverage": 16094, "\u0120Reid": 16095, "\u0120correlation": 16096, "iances": 16097, "avascript": 16098, "\u0120repository": 16099, "retty": 16100, "\u01201972": 16101, "240": 16102, "\u0120oun": 16103, "pol": 16104, "\u0120Reed": 16105, "\u0120tactical": 16106, "isite": 16107, "Apple": 16108, "\u0120Quinn": 16109, "\u0120raped": 16110, "illo": 16111, "Europe": 16112, "\u0120algorithms": 16113, "\u0120Rodrig": 16114, "iu": 16115, "\u0120illum": 16116, "\u0120fame": 16117, "\u0120introducing": 16118, "\u0120delays": 16119, "\u0120Raiders": 16120, "\u0120whistle": 16121, "\u0120novels": 16122, "\u0120Really": 16123, "\u0120deriv": 16124, "\u0120publications": 16125, "\u0120Neither": 16126, "\u0120Commerce": 16127, "\u0120aston": 16128, "language": 16129, "Notes": 16130, "\u0120Roth": 16131, "\u0120Fear": 16132, "\u0120mate": 16133, "\u0120parade": 16134, "\u0120QB": 16135, "\u0120maneu": 16136, "\u0120Cincinnati": 16137, "mitting": 16138, "\u0120waist": 16139, "\u0120Rew": 16140, "\u0120discont": 16141, "\u00d0\u00b0": 16142, "\u0120staring": 16143, "\u0120alias": 16144, "\u0120securities": 16145, "\u0120toilet": 16146, "\u0120Jedi": 16147, "\u0120unlaw": 16148, "vised": 16149, "////////": 16150, "](": 16151, "\u0120Weiss": 16152, "\u0120prest": 16153, "\u0120Compan": 16154, "\u0120memo": 16155, "\u0120Grace": 16156, "July": 16157, "\u0120Elite": 16158, "center": 16159, "\u0120Stay": 16160, "\u0120galaxy": 16161, "\u0120tooth": 16162, "\u0120Settings": 16163, "\u0120subjected": 16164, "\u00e3\u0124\u00a6": 16165, "\u0120lineback": 16166, "\u0120retailers": 16167, "\u0120Want": 16168, "\u0120dangers": 16169, "Air": 16170, "\u0120voluntary": 16171, "eway": 16172, "\u0120interpreted": 16173, "otine": 16174, "\u00c3\u00a7": 16175, "\u0120pel": 16176, "Service": 16177, "\u0120Eventually": 16178, "\u0120careers": 16179, "\u0120threaten": 16180, "\u0120memor": 16181, "\u0120Bradley": 16182, "ancies": 16183, "sn": 16184, "\u0120Unknown": 16185, "National": 16186, "\u0120shadows": 16187, "ailand": 16188, "\u0120Dash": 16189, "Everyone": 16190, "izzard": 16191, "March": 16192, "=(": 16193, "\u0120pulls": 16194, "\u0120stranger": 16195, "\u0120backwards": 16196, "\u0120Bernard": 16197, "imensional": 16198, "\u0120chron": 16199, "\u0120theoretical": 16200, "ktop": 16201, "\u0120ware": 16202, "\u0120Investig": 16203, "\u0120Initi": 16204, "\u0120Operations": 16205, "oven": 16206, "ocide": 16207, "*/": 16208, "\u0120flames": 16209, "\u0120Cash": 16210, "shit": 16211, "\u0120cab": 16212, "\u0120Analy": 16213, "\u0120Seah": 16214, "\u0120defining": 16215, "\u0120ordering": 16216, "\u0120immun": 16217, "\u0120persistent": 16218, "ACH": 16219, "Russian": 16220, "mans": 16221, "\u0120hind": 16222, "\u0120photography": 16223, "\u00c2\u00a9": 16224, "\u0120hug": 16225, "\u0120107": 16226, "\u0120Hence": 16227, "iots": 16228, "udeau": 16229, "\u0120subsidies": 16230, "\u0120routinely": 16231, "\u0120Device": 16232, "itic": 16233, "\u0120disgust": 16234, "lander": 16235, "\u01201940": 16236, "\u0120assignment": 16237, "\u0120Besides": 16238, "wick": 16239, "\u0120Dust": 16240, "usc": 16241, "structed": 16242, "111": 16243, "develop": 16244, "\u0120fond": 16245, "\u0120intersection": 16246, "\u0120dignity": 16247, "\u0120commissioner": 16248, "Without": 16249, "reach": 16250, "\u0120cartoon": 16251, "\u0120scales": 16252, "\u00e3\u0125\u0143": 16253, "FIG": 16254, "\u0120surveys": 16255, "\u0120Indonesia": 16256, "\u0120artwork": 16257, "\u0120unch": 16258, "\u0120cycling": 16259, "unct": 16260, "auer": 16261, "orate": 16262, "\u0120Obviously": 16263, "\u0120characterized": 16264, "feld": 16265, "\u0120affirm": 16266, "\u0120innings": 16267, "\u0120\u00e9": 16268, "\u0120aliens": 16269, "\u0120cloth": 16270, "etooth": 16271, "\u0120Certain": 16272, "\u00c2\u00a7": 16273, "\u0120digest": 16274, "know": 16275, "\u0120XL": 16276, "\u0120predictions": 16277, "\u0120din": 16278, "WAR": 16279, "\u0120aftermath": 16280, "Example": 16281, "\u0120Success": 16282, "\u0120Thr": 16283, "IGN": 16284, "\u0120miner": 16285, "Bus": 16286, "\u0120clarity": 16287, "heimer": 16288, "\u0120OUT": 16289, "\u0120Send": 16290, "\u0120Circle": 16291, "\u0120Diet": 16292, "\u0120pronounced": 16293, "\u0120creators": 16294, "\u0120earthquake": 16295, "attery": 16296, "geons": 16297, "\u0120od": 16298, "\u0120laying": 16299, "orp": 16300, "Ult": 16301, "project": 16302, "\u0120undermin": 16303, "\u0120sequel": 16304, "Sam": 16305, "\u0120Darkness": 16306, "\u0120reception": 16307, "bull": 16308, "YS": 16309, "\u0120Vir": 16310, "\u0120sequences": 16311, "\u0120Coin": 16312, "\u0120outfit": 16313, "\u0120Wait": 16314, "119": 16315, "\u0120delivers": 16316, "......": 16317, "\u0120blown": 16318, "\u0120Esc": 16319, "\u0120Math": 16320, "perm": 16321, "\u0120Ul": 16322, "\u0120glim": 16323, "\u0120facial": 16324, "\u0120greenhouse": 16325, "\u0120tokens": 16326, "/-": 16327, "\u0120Annual": 16328, "\u0120ONE": 16329, "\u0120teenage": 16330, "\u0120Physical": 16331, "\u0120Lang": 16332, "\u0120Celt": 16333, "\u0120sued": 16334, "ividually": 16335, "\u0120patience": 16336, "chair": 16337, "regular": 16338, "\u0120aug": 16339, "inv": 16340, "except": 16341, "\u0120Lil": 16342, "\u0120nest": 16343, "fd": 16344, "sum": 16345, "\u0120Chase": 16346, "Russia": 16347, "\u0120Jennifer": 16348, "\u0120offseason": 16349, "Overall": 16350, "Fore": 16351, "\u0120riot": 16352, "Aud": 16353, "former": 16354, "\u0120defenders": 16355, "\u0120CT": 16356, "iotic": 16357, "ribly": 16358, "\u0120automated": 16359, "\u0120penis": 16360, "\u0120insist": 16361, "\u0120diagram": 16362, "\u0120SQL": 16363, "\u0120Garc": 16364, "\u0120witch": 16365, "client": 16366, "ierra": 16367, "ambers": 16368, "\u0120recount": 16369, "far": 16370, "Very": 16371, "osterone": 16372, "\u0120appreciated": 16373, "\u0120Perfect": 16374, "Section": 16375, "\u0120doses": 16376, "ocaust": 16377, "\u0120costly": 16378, "\u0120grams": 16379, "\u0120Shi": 16380, "\u0120wrestling": 16381, "\u01201971": 16382, "\u0120trophy": 16383, "\u0120nerve": 16384, "\u0120Kaz": 16385, "\u0120Experience": 16386, "\u0120pledged": 16387, "\u0120playback": 16388, "\u0120creativity": 16389, "bye": 16390, "\u0120attackers": 16391, "\u0120holders": 16392, "\u0120Coach": 16393, "\u0120PhD": 16394, "\u0120transfers": 16395, "\u0120colored": 16396, "\u0120Hindu": 16397, "\u0120drown": 16398, "\u0120listened": 16399, "\u0120WA": 16400, "iasm": 16401, "PO": 16402, "\u0120appealing": 16403, "\u0120disclosed": 16404, "\u0120Chicken": 16405, "agging": 16406, "\u0120pleaded": 16407, "\u0120navigation": 16408, "\u0120Returns": 16409, "\u0120[[": 16410, "ROR": 16411, "EA": 16412, "\u0120photographer": 16413, "\u0120Rider": 16414, "ippers": 16415, "\u0120slice": 16416, "\u0120erect": 16417, "\u0120hed": 16418, "issance": 16419, "\u0120Vikings": 16420, "urious": 16421, "\u0120appet": 16422, "oubtedly": 16423, "Child": 16424, "\u0120authentic": 16425, "oos": 16426, "\u0120Making": 16427, "\u0120announcing": 16428, "\u0120bod": 16429, "\u0120meter": 16430, "\u0120Nine": 16431, "\u0120Rogue": 16432, "\u0120workforce": 16433, "\u0120renewed": 16434, "\u0120organisations": 16435, "acs": 16436, "PLE": 16437, "Short": 16438, "\u0120compounds": 16439, "\u0120Visit": 16440, "\u0120envelop": 16441, "earth": 16442, "\u0120supportive": 16443, "ggle": 16444, "\u0120Brussels": 16445, "\u0120Guild": 16446, "Create": 16447, "REL": 16448, "\u0120averaged": 16449, "\u01201969": 16450, "riages": 16451, "\u0120lengthy": 16452, "\u0120forgot": 16453, "Okay": 16454, "\u0120Erd": 16455, "\u0120dealer": 16456, "\u0120recession": 16457, "DD": 16458, "\u0120desperately": 16459, "\u0120hunger": 16460, "\u0120sticks": 16461, "\u0120mph": 16462, "\u0120Faith": 16463, "\u0120intentionally": 16464, "\u0120demol": 16465, "ueller": 16466, "\u0120Sale": 16467, "\u0120debris": 16468, "spring": 16469, "\u0120leap": 16470, ">>>>": 16471, "\u0120containers": 16472, "selling": 16473, "ranean": 16474, "attering": 16475, "\u0120commented": 16476, "\u0120CM": 16477, "onut": 16478, "\u0120woods": 16479, "especially": 16480, "\u0120organize": 16481, "ivic": 16482, "\u0120Woods": 16483, "anga": 16484, "squ": 16485, "\u0120maj": 16486, "amon": 16487, "\u0120axis": 16488, "\u01201974": 16489, "\u0120Denmark": 16490, "\u0120warrior": 16491, "\u0120Pand": 16492, "\u0120outlined": 16493, "\u0120BO": 16494, "insula": 16495, "zilla": 16496, "ebook": 16497, "\u0120dare": 16498, "\u0120searched": 16499, "\u0120navigate": 16500, "Sn": 16501, "writing": 16502, "\u0120united": 16503, "Japan": 16504, "\u0120Hebrew": 16505, "\u0120flame": 16506, "\u0120relies": 16507, "\u0120catching": 16508, "\u0120Sho": 16509, "\u0120imprisonment": 16510, "\u0120pockets": 16511, "\u0120closure": 16512, "\u0120Fam": 16513, "tim": 16514, "adequ": 16515, "Activity": 16516, "\u0120recruiting": 16517, "\u0120WATCH": 16518, "\u0120Argentina": 16519, "dest": 16520, "\u0120apologize": 16521, "oro": 16522, "\u0120lacks": 16523, "\u0120tuned": 16524, "\u0120Griffin": 16525, "\u0120infamous": 16526, "\u0120celebrity": 16527, "sson": 16528, "\u0120----------------------------------------------------------------": 16529, "\u0120Isis": 16530, "\u0120Display": 16531, "\u0120credibility": 16532, "\u0120economies": 16533, "\u0120headline": 16534, "\u0120Cowboys": 16535, "\u0120indef": 16536, "\u0120lately": 16537, "\u0120incentives": 16538, "button": 16539, "\u0120Mob": 16540, "Aut": 16541, "\u0120resigned": 16542, "\u0120Om": 16543, "camp": 16544, "\u0120profiles": 16545, "\u0120schemes": 16546, "olphins": 16547, "ayed": 16548, "Clinton": 16549, "enh": 16550, "\u0120Yahoo": 16551, "\u0120abst": 16552, "\u0120ank": 16553, "suits": 16554, "\u0120wished": 16555, "\u0120Marco": 16556, "udden": 16557, "\u0120sphere": 16558, "\u0120Bishop": 16559, "\u0120incorporated": 16560, "\u0120Plant": 16561, "114": 16562, "\u0120hated": 16563, "pic": 16564, "\u0120donate": 16565, "\u0120lined": 16566, "\u0120beans": 16567, "\u0120stealing": 16568, "\u0120costume": 16569, "\u0120sheriff": 16570, "\u0120forty": 16571, "\u0120intact": 16572, "\u0120adapted": 16573, "\u0120travelling": 16574, "bart": 16575, "\u0120nicely": 16576, "\u0120dried": 16577, "\u0120scal": 16578, "osity": 16579, "NOTE": 16580, "\u0120Bh": 16581, "\u0120Broncos": 16582, "\u0120Ign": 16583, "\u0120intimate": 16584, "\u0120chemistry": 16585, "\u0120optimal": 16586, "Deb": 16587, "\u0120Generation": 16588, "\u0120],": 16589, "ichi": 16590, "\u0120Wii": 16591, "\u0120YOUR": 16592, "ventions": 16593, "Write": 16594, "\u0120popul": 16595, "unning": 16596, "\u0120Wor": 16597, "Vol": 16598, "\u0120queen": 16599, "heads": 16600, "KK": 16601, "\u0120analyze": 16602, "opic": 16603, "earchers": 16604, "\u0120dot": 16605, "legraph": 16606, "astically": 16607, "\u0120upgrades": 16608, "\u0120cares": 16609, "\u0120extending": 16610, "\u0120freeze": 16611, "\u0120inability": 16612, "\u0120organs": 16613, "\u0120pretend": 16614, "\u0120outlet": 16615, "113": 16616, "olan": 16617, "\u0120Mall": 16618, "uling": 16619, "talk": 16620, "\u0120expressing": 16621, "\u0120Always": 16622, "\u0120Begin": 16623, "files": 16624, "\u0120licenses": 16625, "%%": 16626, "\u0120Mitt": 16627, "\u0120filters": 16628, "\u0120Milwaukee": 16629, "GN": 16630, "\u0120unfold": 16631, "Mo": 16632, "\u0120nutrition": 16633, "ppo": 16634, "Bo": 16635, "\u0120founding": 16636, "\u0120undermine": 16637, "\u0120easiest": 16638, "\u0120Czech": 16639, "\u0120Mack": 16640, "\u0120sexuality": 16641, "\u0120Nixon": 16642, "Win": 16643, "\u0120Arn": 16644, "\u0120Kin": 16645, "\u00e3\u0124\u00a3": 16646, "icer": 16647, "\u0120fortun": 16648, "\u0120surfaces": 16649, "aghd": 16650, "\u0120carriers": 16651, "\u0120PART": 16652, "\u0120Tib": 16653, "\u0120interval": 16654, "\u0120frustrating": 16655, "\u0120Ship": 16656, "\u0120Armed": 16657, "ffe": 16658, "\u0120boats": 16659, "\u0120Abraham": 16660, "inis": 16661, "\u0120suited": 16662, "thread": 16663, "iov": 16664, "abul": 16665, "\u0120Venezuela": 16666, "\u0120tom": 16667, "super": 16668, "\u0120castle": 16669, "although": 16670, "ioxide": 16671, "eches": 16672, "\u0120evolutionary": 16673, "\u0120negotiate": 16674, "\u0120confronted": 16675, "Remember": 16676, "\u0120170": 16677, "Such": 16678, "\u0120911": 16679, "mult": 16680, "\u0120Abyss": 16681, "urry": 16682, "kees": 16683, "spec": 16684, "\u0120Barbara": 16685, "\u0120belonging": 16686, "\u0120villain": 16687, "istani": 16688, "\u0120accountable": 16689, "\u0120portions": 16690, "\u0120Decl": 16691, "Ur": 16692, "\u0120Kate": 16693, "gre": 16694, "\u0120magazines": 16695, "UCK": 16696, "\u0120regulate": 16697, "omon": 16698, "\u0120Almost": 16699, "\u0120overview": 16700, "\u0120scram": 16701, "\u0120loot": 16702, "\u0120Fitz": 16703, "\u0120characteristic": 16704, "\u0120Snake": 16705, "say": 16706, "\u0120Rico": 16707, "\u0120trait": 16708, "\u0120Joined": 16709, "aucus": 16710, "\u0120adaptation": 16711, "\u0120Airlines": 16712, "\u0120archae": 16713, "\u0120Ide": 16714, "\u0120bikes": 16715, "\u0120literary": 16716, "\u0120influences": 16717, "\u0120Used": 16718, "Creat": 16719, "\u0120plea": 16720, "\u0120Defence": 16721, "\u0120Assass": 16722, "\u0120pond": 16723, "ULT": 16724, ")\"": 16725, "\u0120evaluated": 16726, "\u0120obtaining": 16727, "\u0120demographic": 16728, "\u0120vigil": 16729, "aley": 16730, "\u0120spouse": 16731, "\u0120Seahawks": 16732, "respons": 16733, "\u0120Belt": 16734, "umatic": 16735, "\u0120rises": 16736, "runner": 16737, "\u0120Michelle": 16738, "\u0120potent": 16739, "race": 16740, "\u0120PAC": 16741, "Find": 16742, "olesterol": 16743, "ISS": 16744, "\u0120Introduced": 16745, "resses": 16746, "ignment": 16747, "Os": 16748, "\u0120Tu": 16749, "\u0120Dex": 16750, "icides": 16751, "\u0120sparked": 16752, "\u0120Laura": 16753, "\u0120Bryant": 16754, "\u0120smiling": 16755, "\u0120Nexus": 16756, "\u0120defendants": 16757, "\u0120Catal": 16758, "\u0120dishes": 16759, "shaped": 16760, "\u0120prolong": 16761, "mt": 16762, "($": 16763, "\u00e3\u0122\u0124": 16764, "\u0120calculations": 16765, "\u0120Same": 16766, "\u0120piv": 16767, "HH": 16768, "\u0120cancelled": 16769, "\u0120grin": 16770, "\u0120territories": 16771, "istically": 16772, "Come": 16773, "\u0120Parent": 16774, "Project": 16775, "\u0120neglig": 16776, "\u0120Privacy": 16777, "\u0120ammo": 16778, "LECT": 16779, "olutely": 16780, "\u0120Epic": 16781, "\u0120misunder": 16782, "wal": 16783, "April": 16784, "mos": 16785, "pathy": 16786, "\u0120Carson": 16787, "\u0120albums": 16788, "\u0120Easy": 16789, "\u0120pistol": 16790, "<<": 16791, "\u0120\\(": 16792, "target": 16793, "help": 16794, "\u0120interpre": 16795, "conscious": 16796, "\u0120Housing": 16797, "\u0120Joint": 16798, "127": 16799, "\u0120beers": 16800, "science": 16801, "\u0120Firefox": 16802, "effective": 16803, "\u0120Cabin": 16804, "\u0120Okay": 16805, "\u0120Applic": 16806, "\u0120spacecraft": 16807, "\u0120SR": 16808, "vet": 16809, "\u0120Strange": 16810, "SB": 16811, "\u0120corps": 16812, "iberal": 16813, "efficient": 16814, "\u0120prevalence": 16815, "\u0120economists": 16816, "118": 16817, "Thread": 16818, "ordable": 16819, "ODE": 16820, "\u0120Cant": 16821, "=-=-": 16822, "ifiable": 16823, "\u0120Around": 16824, "\u0120pole": 16825, "\u0120willingness": 16826, "CLA": 16827, "\u0120Kid": 16828, "\u0120complement": 16829, "\u0120scattered": 16830, "\u0120inmates": 16831, "\u0120bleeding": 16832, "every": 16833, "\u0120queue": 16834, "\u0120Train": 16835, "\u0120hij": 16836, "\u0120melee": 16837, "pleted": 16838, "\u0120digit": 16839, "\u0120gem": 16840, "official": 16841, "\u0120lifting": 16842, "\u00d0\u00b5": 16843, "Requ": 16844, "itutes": 16845, "\u0120packaging": 16846, "\u0120Workers": 16847, "hran": 16848, "\u0120Lebanon": 16849, "olesc": 16850, "\u0120punished": 16851, "\u0120Juan": 16852, "\u0120jam": 16853, "\u0120Document": 16854, "\u0120mapping": 16855, "icates": 16856, "\u0120inevitably": 16857, "\u0120vanilla": 16858, "\u0120Ton": 16859, "\u0120watches": 16860, "\u0120leagues": 16861, "\u0120initiated": 16862, "degree": 16863, "portion": 16864, "\u0120recalls": 16865, "\u0120ruin": 16866, "\u0120melt": 16867, "IAN": 16868, "\u0120hem": 16869, "Exp": 16870, "\u0120baking": 16871, "\u0120Colomb": 16872, "atible": 16873, "\u0120radius": 16874, "plug": 16875, "\u0120IF": 16876, "etically": 16877, "\u0120fict": 16878, "HER": 16879, "\u0120Tap": 16880, "atinum": 16881, "\u0120ink": 16882, "\u0120coh": 16883, "\u0120Wizard": 16884, "both": 16885, "tex": 16886, "\u0120spends": 16887, "\u0120Currently": 16888, "\u0120Pit": 16889, "\u0120neurons": 16890, "ignt": 16891, "\u0120rall": 16892, "\u0120buses": 16893, "building": 16894, "\u0120adjustments": 16895, "\u0120cried": 16896, "iblical": 16897, "atted": 16898, "\u0120Zion": 16899, "\u0120Matter": 16900, "\u0120meditation": 16901, "\u0120Dennis": 16902, "\u0120ours": 16903, "\u0120Tab": 16904, "\u0120rankings": 16905, "ortal": 16906, "\u0120advers": 16907, "\u0120surrender": 16908, "\u0120Gob": 16909, "cium": 16910, "omas": 16911, "imeter": 16912, "\u0120multiplayer": 16913, "\u0120heroin": 16914, "\u0120optimistic": 16915, "\u0120indicator": 16916, "\u0120Brig": 16917, "\u0120grocery": 16918, "\u0120applicant": 16919, "\u0120Rocket": 16920, "vid": 16921, "Exception": 16922, "pent": 16923, "\u0120organizing": 16924, "\u0120encounters": 16925, "\u0120TOD": 16926, "\u0120jewel": 16927, "Save": 16928, "\u0120Christie": 16929, "\u0120heating": 16930, "\u0120lazy": 16931, "\u0120CP": 16932, "\u0120cousin": 16933, "Config": 16934, "\u0120regener": 16935, "\u0120nearest": 16936, "\u0120achieving": 16937, "ENS": 16938, "throw": 16939, "\u0120Richmond": 16940, "antle": 16941, "2002": 16942, "\u0120anten": 16943, "bird": 16944, "133": 16945, "\u0120narc": 16946, "raint": 16947, "unny": 16948, "\u0120Hispanic": 16949, "ournaments": 16950, "\u0120prophe": 16951, "\u0120Thailand": 16952, "\u0120Ti": 16953, "\u0120injection": 16954, "\u0120inherit": 16955, "ravis": 16956, "\u0120medi": 16957, "\u0120whoever": 16958, "\u0120DEBUG": 16959, "GP": 16960, "\u0120Hud": 16961, "Card": 16962, "prom": 16963, "\u0120por": 16964, "\u0120overhead": 16965, "Law": 16966, "\u0120violate": 16967, "\u0120heated": 16968, "\u0120descriptions": 16969, "\u0120achievements": 16970, "\u0120Beer": 16971, "\u0120Quant": 16972, "Was": 16973, "\u0120eighth": 16974, "\u0120Iv": 16975, "\u0120specialized": 16976, "UPDATE": 16977, "\u0120Delta": 16978, "Pop": 16979, "Jul": 16980, "\u0120Ask": 16981, "ophy": 16982, "\u0120newsletters": 16983, "\u0120Tool": 16984, "\u0120gard": 16985, "\u0120Confeder": 16986, "\u0120GMT": 16987, "\u0120Abbott": 16988, "\u0120immunity": 16989, "\u0120VM": 16990, "Islam": 16991, "\u0120implicit": 16992, "wd": 16993, "\u01201944": 16994, "ravity": 16995, "ometric": 16996, "\u0120surviving": 16997, "urai": 16998, "\u0120Prison": 16999, "\u0120rust": 17000, "\u0120Sketch": 17001, "\u0120bees": 17002, "\u0120Theory": 17003, "\u0120merit": 17004, "Tex": 17005, "chat": 17006, "\u0120mim": 17007, "\u0120paste": 17008, "\u0120Koch": 17009, "\u0120ignorance": 17010, "\u0120Shoot": 17011, "\u0120basement": 17012, "United": 17013, "\u0120Advis": 17014, "height": 17015, "\u0120foster": 17016, "\u0120detain": 17017, "information": 17018, "\u0120neural": 17019, "';": 17020, "\u0120proves": 17021, "allery": 17022, "\u0120invitation": 17023, "umbers": 17024, "\u0120cattle": 17025, "\u0120bicycle": 17026, "zi": 17027, "\u0120consultant": 17028, "\u0120apology": 17029, "\u0120Tiger": 17030, "\u0120123": 17031, "999": 17032, "\u0120individually": 17033, "rt": 17034, "igion": 17035, "\u0120Brazilian": 17036, "\u0120disturb": 17037, "\u0120entrepreneurs": 17038, "\u0120forests": 17039, "cerpt": 17040, "plates": 17041, "pher": 17042, "clipse": 17043, "\u0120twitter": 17044, "\u0120acids": 17045, "ographical": 17046, "hum": 17047, "\u0120Bald": 17048, "ifully": 17049, "\u0120compiler": 17050, "\u0120DA": 17051, "\u0120donor": 17052, "asi": 17053, "\u0120tribal": 17054, "lash": 17055, "\u0120Config": 17056, "\u0120applicants": 17057, "\u0120salaries": 17058, "135": 17059, "Putin": 17060, "\u0120Focus": 17061, "irs": 17062, "\u0120misconduct": 17063, "\u0120Haz": 17064, "\u0120eaten": 17065, "Mobile": 17066, "Muslim": 17067, "\u0120Marcus": 17068, "viol": 17069, "\u0120favorable": 17070, "\u0120stub": 17071, "adin": 17072, "\u0120Hob": 17073, "\u0120faithful": 17074, "\u0120electronics": 17075, "\u0120vacuum": 17076, "wait": 17077, "backed": 17078, "economic": 17079, "dist": 17080, "\u0120tenure": 17081, "\u0120sincere": 17082, "\u0120Together": 17083, "\u0120Wave": 17084, "\u0120progression": 17085, "\u0120denying": 17086, "\u0120distress": 17087, "braska": 17088, "third": 17089, "\u0120mixing": 17090, "\u0120colonial": 17091, "\u0120privately": 17092, "\u0120unrest": 17093, "aternity": 17094, "\u0120premises": 17095, "anti": 17096, "gregation": 17097, "\u0120licence": 17098, "\u0120Hind": 17099, "\u0120Samuel": 17100, "\u0120convincing": 17101, "\u0120Ace": 17102, "\u0120Rust": 17103, "\u0120Netanyahu": 17104, "\u0120handles": 17105, "\u0120Patch": 17106, "oriented": 17107, "aho": 17108, "\u0120Gonz": 17109, "\u0120hackers": 17110, "claimer": 17111, "\u0120customs": 17112, "\u0120Gran": 17113, "fighters": 17114, "\u0120luc": 17115, "\u0120manuscript": 17116, "arenthood": 17117, "\u0120devil": 17118, "\u0120warriors": 17119, "\u0120offenders": 17120, "William": 17121, "\u0120holidays": 17122, "\u0120nightmare": 17123, "\u0120lever": 17124, "ifferent": 17125, "Stat": 17126, "\u0120exhibition": 17127, "puted": 17128, "\u0120Pure": 17129, "\u0120alpha": 17130, "\u0120enthusiasm": 17131, "\u0120Representatives": 17132, "EAR": 17133, "\u0120Typ": 17134, "\u0120wheat": 17135, "\u0120Alf": 17136, "\u0120correction": 17137, "\u0120evangel": 17138, "ATT": 17139, "Miss": 17140, "\u0120soup": 17141, "\u0120implied": 17142, "param": 17143, "\u0120sexy": 17144, "\u0120Lux": 17145, "\u0120republic": 17146, "patch": 17147, "ablish": 17148, "\u0120icons": 17149, "\u0120fathers": 17150, "\u0120GET": 17151, "\u0120Carib": 17152, "\u0120regulated": 17153, "\u0120Cohen": 17154, "\u0120Bobby": 17155, "\u0120ner": 17156, "\u0120bent": 17157, "ventory": 17158, "\u0120Along": 17159, "\u0120EST": 17160, "\u0120Wallace": 17161, "\u0120murders": 17162, "rise": 17163, "kell": 17164, "\u0120Commonwealth": 17165, "\u0120nasty": 17166, "eta": 17167, "\u0120MIT": 17168, "\u0120administered": 17169, "\u0120genuinely": 17170, "Editor": 17171, "nick": 17172, "\u0120hydro": 17173, "********************************": 17174, "\u0120Ble": 17175, "\u0120fines": 17176, "\u0120gorge": 17177, "ausible": 17178, "rh": 17179, "\u0120apple": 17180, "mentioned": 17181, "\u0120rope": 17182, "otyp": 17183, "HR": 17184, "\u0120disappointing": 17185, "\u0120cage": 17186, "nik": 17187, "\u0120doubts": 17188, "\u0120FREE": 17189, "prints": 17190, "\u0120MUST": 17191, "\u0120vendors": 17192, "\u0120Inqu": 17193, "\u0120liberals": 17194, "\u0120contractor": 17195, "\u0120upside": 17196, "children": 17197, "\u0120tricky": 17198, "\u0120regulators": 17199, "charged": 17200, "liter": 17201, "\u0120***": 17202, "\u0120rebell": 17203, "lang": 17204, "\u0120locals": 17205, "\u0120physicians": 17206, "\u0120hey": 17207, "arse": 17208, "tm": 17209, "\u0120Lex": 17210, "\u0120behavioral": 17211, "successful": 17212, "FX": 17213, "\u0120brick": 17214, "ovic": 17215, "\u0120conform": 17216, "\u0120reviewing": 17217, "\u0120insights": 17218, "\u0120biology": 17219, "\u0120Remove": 17220, "\u0120Extra": 17221, "\u0120committing": 17222, "induced": 17223, "ignty": 17224, "igm": 17225, "\u0120atomic": 17226, "Common": 17227, "\u0120EM": 17228, "\u0120Pere": 17229, "\u0120Items": 17230, "eh": 17231, "\u0120preserved": 17232, "\u0120Hood": 17233, "\u0120prisoner": 17234, "\u0120bankruptcy": 17235, "\u0120gren": 17236, "ushes": 17237, "\u0120exploitation": 17238, "\u0120signatures": 17239, "\u0120finan": 17240, "],\"": 17241, "\u0120MR": 17242, "\u0120meg": 17243, "remlin": 17244, "\u0120musicians": 17245, "\u0120selecting": 17246, "\u0120examining": 17247, "INK": 17248, "lated": 17249, "Hi": 17250, "\u0120artic": 17251, "\u0120pets": 17252, "\u0120impair": 17253, "\u0120MAN": 17254, "\u0120tablets": 17255, "include": 17256, "Range": 17257, "\u0120caut": 17258, "\u0120logs": 17259, "\u0120mounting": 17260, "\u0120unaware": 17261, "\u0120dynamics": 17262, "\u0120Palestine": 17263, "\u0120Quarter": 17264, "\u0120Purple": 17265, "\u0120ma": 17266, "\u0120Import": 17267, "\u0120collections": 17268, "ciation": 17269, "\u0120successor": 17270, "\u0120clone": 17271, "\u0120aiming": 17272, "\u0120possessed": 17273, "\u0120sticking": 17274, "\u0120shaking": 17275, "\u0120locate": 17276, "\u0120Hockey": 17277, "Turn": 17278, "170": 17279, "\u0120fifteen": 17280, "\u0120Harrison": 17281, "\u0120continuously": 17282, "\u0120TC": 17283, "\u0120Valent": 17284, "\u0120Rescue": 17285, "\u0120bypass": 17286, "amount": 17287, "\u0120mast": 17288, "\u0120protects": 17289, "\u0120artistic": 17290, "\u0120sometime": 17291, "\u0120shoe": 17292, "\u0120shouted": 17293, "ificant": 17294, "etitive": 17295, "\u0120Register": 17296, "\u0120Jin": 17297, "\u0120concentrated": 17298, "lington": 17299, "onies": 17300, "\u0120generator": 17301, "yrim": 17302, "\u0120Armen": 17303, "\u0120clearing": 17304, "ido": 17305, "\u0120TW": 17306, "alph": 17307, "\u0120ladies": 17308, "Hard": 17309, "\u0120dialog": 17310, "\u0120inputs": 17311, "\u00e6\u013e": 17312, "\u0120poses": 17313, "\u0120slots": 17314, "\u0120Premium": 17315, "\u0120leaks": 17316, "\u0120bosses": 17317, "\u0120113": 17318, "course": 17319, "Acc": 17320, "\u0120Newton": 17321, "\u0120Austria": 17322, "\u0120Mage": 17323, "\u0120teaches": 17324, "abad": 17325, "\u0120wears": 17326, "\u0120cyl": 17327, "\u0120curse": 17328, "\u0120Sales": 17329, "\u0120Wings": 17330, "\u0120psy": 17331, "\u0120gaps": 17332, "\u0120Iceland": 17333, "\u0120Pinterest": 17334, "\u0120landlord": 17335, "\u0120definitions": 17336, "\u0120Ker": 17337, "\u0120sufficiently": 17338, "\u0120Pence": 17339, "\u0120Architect": 17340, "\u0120surpass": 17341, "\u0120114": 17342, "\u0120superhero": 17343, "\u0120Disease": 17344, "\u0120priests": 17345, "\u0120Culture": 17346, "\u0120definitive": 17347, "\u0120secretly": 17348, "\u0120Dance": 17349, "install": 17350, "chief": 17351, "\u0120Jessica": 17352, "Would": 17353, "Updated": 17354, "\u0120locker": 17355, "\u0120Kay": 17356, "\u0120memorial": 17357, "\u00e8\u00a6": 17358, "fat": 17359, "\u0120disgu": 17360, "\u0120flavors": 17361, "\u0120Baseball": 17362, "\u0120Resistance": 17363, "\u0120kicks": 17364, "\u0120env": 17365, "\u0120teenagers": 17366, "Dark": 17367, "\u0120CAR": 17368, "\u0120halt": 17369, "\u0120LG": 17370, "\u0120Gabriel": 17371, "\u0120fever": 17372, "\u0120satur": 17373, "\u0120mall": 17374, "\u0120affiliate": 17375, "\u0120Sleep": 17376, "\u0120Specific": 17377, "\u0120Vel": 17378, "\u0120jar": 17379, "\u0120Sacred": 17380, "\u0120Edwards": 17381, "\u0120ACL": 17382, "\u0120retained": 17383, "\u0120Giant": 17384, "\u0120limitation": 17385, "inces": 17386, "\u0120refusal": 17387, "\u0120Tale": 17388, "\u0120Butler": 17389, "\u0120accidents": 17390, "\u0120CSS": 17391, "\u0120imported": 17392, "\u0120Copy": 17393, "\u00ce\u00b1": 17394, "ERT": 17395, "zel": 17396, "\u0120divisions": 17397, "hots": 17398, "\u0120Alb": 17399, "\u0120DS": 17400, "Loader": 17401, "Washington": 17402, "atisf": 17403, "\u0120Creative": 17404, "\\.": 17405, "\u0120Autom": 17406, "redict": 17407, "\u0120receptor": 17408, "\u0120Carlos": 17409, "Method": 17410, "oka": 17411, "\u0120malicious": 17412, "\u0120stepping": 17413, ",[": 17414, "\u0120Dad": 17415, "\u0120attraction": 17416, "\u0120Effects": 17417, "\u0120Pirate": 17418, "\u0120Cer": 17419, "\u0120Industry": 17420, "\u0120Rud": 17421, "\u0120charter": 17422, "\u0120dining": 17423, "\u0120insists": 17424, "\u0120configure": 17425, "\u0120(#": 17426, "\u0120Simple": 17427, "\u0120Scroll": 17428, "UTC": 17429, "175": 17430, "\u0120Kon": 17431, "\u0120marketplace": 17432, "\u0120\u00e3\u0124": 17433, "\u0120refres": 17434, "\u0120gates": 17435, "erred": 17436, "\u0120Pod": 17437, "\u0120behave": 17438, "Frank": 17439, "node": 17440, "\u0120endorsed": 17441, "hett": 17442, "asive": 17443, "\u0120Homeland": 17444, "\u0120rides": 17445, "\u0120Leave": 17446, "erness": 17447, "\u0120flooding": 17448, "AFP": 17449, "\u0120risen": 17450, "\u0120continually": 17451, "\u0120unanim": 17452, "\u0120Contract": 17453, "\u0120Pas": 17454, "\u0120guided": 17455, "\u0120Chile": 17456, "bd": 17457, "\u0120succ": 17458, "ptic": 17459, "\u0120committees": 17460, "\u0120Luther": 17461, "\u0120Anyone": 17462, "\u0120sab": 17463, "124": 17464, "\u0120pixel": 17465, "\u0120Bak": 17466, "\u0120Tag": 17467, "\u0120Bennett": 17468, "Enter": 17469, "small": 17470, "\u0120Presidential": 17471, "\u0120pul": 17472, "\u0120contrace": 17473, "archive": 17474, "\u0120coastal": 17475, "\u0120Kids": 17476, "192": 17477, "\u00e2\u0122\u00b2": 17478, "icky": 17479, "INGTON": 17480, "\u0120wolf": 17481, "\u0120Stalin": 17482, "Tur": 17483, "idget": 17484, "amas": 17485, "\u0120Unless": 17486, "\u0120sponsor": 17487, "\u0120morph": 17488, "\u0120Choose": 17489, "\u0120runner": 17490, "\u0120unbel": 17491, "\u0120mud": 17492, "\u0120Mana": 17493, "\u0120dubbed": 17494, "\u0120godd": 17495, "urers": 17496, "window": 17497, "\u0120relied": 17498, "\u0120celebrating": 17499, "osc": 17500, "\u0120135": 17501, "\u0120lobbying": 17502, "\u0120incomplete": 17503, "\u0120restriction": 17504, "\u0120incap": 17505, "itus": 17506, "\u0120expectation": 17507, "\u0120Apollo": 17508, "\u0120intens": 17509, "\u0120sync": 17510, "GH": 17511, "\u0120manipulation": 17512, "BY": 17513, "\u0120spear": 17514, "\u0120breasts": 17515, "\u0120volcan": 17516, "ilia": 17517, "Material": 17518, "\u0120formats": 17519, "\u0120Bast": 17520, "\u0120parliamentary": 17521, "\u0120snake": 17522, "\u0120servants": 17523, "\u0120Trudeau": 17524, "\u0120Grim": 17525, "\u0120Arabic": 17526, "\u0120SCP": 17527, "\u0120Boys": 17528, "station": 17529, "\u0120prospective": 17530, "orde": 17531, "initialized": 17532, "\u0120bored": 17533, "ABLE": 17534, "\u0120accessed": 17535, "\u0120taxi": 17536, "\u0120Shell": 17537, "aiden": 17538, "ursed": 17539, "inates": 17540, "\u0120Insurance": 17541, "\u0120Pete": 17542, "September": 17543, "650": 17544, "\u0120adventures": 17545, "\u0120Cover": 17546, "\u0120tribute": 17547, "\u0120sketch": 17548, "\u0120empower": 17549, "\u0120\u00d8": 17550, "\u0120Glenn": 17551, "\u0120Daw": 17552, "=\\\"": 17553, "\u0120Politics": 17554, "\u0120guides": 17555, "\u0120dioxide": 17556, "\u0120Gore": 17557, "\u0120Bright": 17558, "\u0120Sierra": 17559, "\u0120valued": 17560, "cond": 17561, "\u0120pointer": 17562, "Select": 17563, "\u0120risky": 17564, "\u0120absorb": 17565, "images": 17566, "\u0120refuses": 17567, "\u0120bonuses": 17568, "___": 17569, "\u0120hilar": 17570, "\u0120Features": 17571, "220": 17572, "\u0120Collector": 17573, "Foot": 17574, "\u01201964": 17575, "culus": 17576, "\u0120dawn": 17577, "\u0120workout": 17578, "\u0120LO": 17579, "\u0120philosophical": 17580, "\u0120Sandy": 17581, "\u0120Youth": 17582, "\u0120liable": 17583, "Af": 17584, "blue": 17585, "\u0120overturn": 17586, "lessness": 17587, "\u0120Tribune": 17588, "\u0120Ing": 17589, "\u0120factories": 17590, "\u0120catches": 17591, "\u0120prone": 17592, "\u0120matrix": 17593, "\u0120login": 17594, "\u0120inacc": 17595, "\u0120exert": 17596, "sys": 17597, "\u0120needle": 17598, "\u0120Qur": 17599, "\u0120notified": 17600, "oulder": 17601, "tx": 17602, "\u0120reminds": 17603, "\u0120publishers": 17604, "\u0120nort": 17605, "\u0120git": 17606, "\u0120flies": 17607, "\u0120Emily": 17608, "\u0120flowing": 17609, "\u0120Alien": 17610, "\u0120Strateg": 17611, "\u0120hardest": 17612, "\u0120modification": 17613, "API": 17614, "\u0120MY": 17615, "\u0120crashes": 17616, "stairs": 17617, "number": 17618, "\u0120urging": 17619, "channel": 17620, "\u0120Falcon": 17621, "\u0120inhabitants": 17622, "\u0120terrifying": 17623, "\u0120utilize": 17624, "\u0120banner": 17625, "\u0120cigarettes": 17626, "\u0120senses": 17627, "\u0120Holmes": 17628, "\u0120practition": 17629, "\u0120Phillips": 17630, "otto": 17631, "\u0120compile": 17632, "Model": 17633, "\u0120Ko": 17634, "\u0120[]": 17635, "Americans": 17636, "\u0120Terms": 17637, "\u0120medications": 17638, "\u0120Ana": 17639, "\u0120fundamentally": 17640, "\u0120Notice": 17641, "\u0120weaker": 17642, "\u01200000": 17643, "\u0120garlic": 17644, "\u0120outbreak": 17645, "\u0120economist": 17646, "\u0120Birth": 17647, "\u0120obstacles": 17648, "arcer": 17649, "\u0120Orthodox": 17650, "\u0120placebo": 17651, "\u0120Crew": 17652, "aspberry": 17653, "\u0120Angels": 17654, "\u0120discharge": 17655, "\u0120destructive": 17656, "117": 17657, "\u0120Rising": 17658, "\u0120dairy": 17659, "late": 17660, "\u0120collision": 17661, "\u0120Tigers": 17662, "eanor": 17663, "ocumented": 17664, "\u0120Invalid": 17665, "\u0120dont": 17666, "\u0120Liter": 17667, "\u0120Va": 17668, "\u0120hydrogen": 17669, "\u0120variants": 17670, "\u0120Browns": 17671, "\u01201965": 17672, "\u0120indigenous": 17673, "\u0120trades": 17674, "\u0120remainder": 17675, "\u0120swept": 17676, "\u0120Impact": 17677, "\u0120redist": 17678, "\u0120unint": 17679, "graduate": 17680, "\u00e3\u0125\u0137": 17681, "\u0120WILL": 17682, "\u00e3\u0123\u00ae\u00e7": 17683, "\u0120Critical": 17684, "\u0120fisher": 17685, "\u0120vicious": 17686, "\u0120reversed": 17687, "Year": 17688, "\u0120Sox": 17689, "\u0120shootings": 17690, "\u0120filming": 17691, "\u0120touchdowns": 17692, "aires": 17693, "mel": 17694, "\u0120grandfather": 17695, "\u0120affection": 17696, "ingle": 17697, "\u0120overly": 17698, "Additional": 17699, "\u0120supreme": 17700, "\u0120Grad": 17701, "\u0120sporting": 17702, "\u0120mercy": 17703, "\u0120Brooks": 17704, "ounty": 17705, "\u0120performs": 17706, "\u0120tightly": 17707, "\u0120demons": 17708, "\u0120killings": 17709, "\u0120faction": 17710, "\u0120Nova": 17711, "auts": 17712, "\u0120undoubtedly": 17713, "arin": 17714, "\u0120underway": 17715, "rak": 17716, "\u0120liv": 17717, "\u0120Region": 17718, "\u0120briefing": 17719, "sers": 17720, "cloud": 17721, "\u0120Mik": 17722, "usp": 17723, "\u0120prediction": 17724, "azor": 17725, "\u0120portable": 17726, "\u0120Gand": 17727, "\u0120presenting": 17728, "\u01201080": 17729, "\u00c2\u00bb": 17730, "ushi": 17731, "\u0120Spark": 17732, "thereum": 17733, "\u0120justification": 17734, "\u0120Ny": 17735, "\u0120contractors": 17736, "mingham": 17737, "\u0120Style": 17738, "\u00e5\u0127": 17739, "\u0120Chronicles": 17740, "\u0120Picture": 17741, "\u0120proving": 17742, "\u0120wives": 17743, "sett": 17744, "\u0120molecules": 17745, "\u0120Fairy": 17746, "\u0120consisting": 17747, "\u0120pier": 17748, "alone": 17749, "inition": 17750, "\u0120nucle": 17751, "json": 17752, "\u0120gotta": 17753, "\u0120mobil": 17754, "\u0120verbal": 17755, "arium": 17756, "\u0120monument": 17757, "ucked": 17758, "\u0120256": 17759, "Tech": 17760, "minecraft": 17761, "\u0120Track": 17762, "\u0120tile": 17763, "\u0120compatibility": 17764, "asis": 17765, "\u0120sadd": 17766, "\u0120instructed": 17767, "\u0120Mueller": 17768, "\u0120lethal": 17769, "\u0120hormone": 17770, "\u0120orche": 17771, "else": 17772, "\u0120skelet": 17773, "\u0120entertaining": 17774, "\u0120minimize": 17775, "again": 17776, "\u0120undergo": 17777, "\u0120constraints": 17778, "\u0120cigarette": 17779, "\u0120Islamist": 17780, "\u0120travels": 17781, "\u0120Panthers": 17782, "lings": 17783, "Care": 17784, "\u0120lawsuits": 17785, "uras": 17786, "\u0120cryst": 17787, "\u0120lowered": 17788, "\u0120aerial": 17789, "\u0120combinations": 17790, "\u0120haun": 17791, "\u0120cha": 17792, "\u0120vine": 17793, "\u0120quantities": 17794, "\u0120linking": 17795, "bank": 17796, "\u0120soy": 17797, "Bill": 17798, "\u0120Angela": 17799, "\u0120recipient": 17800, "\u0120Protest": 17801, "\u0120socket": 17802, "\u0120solidarity": 17803, "\u0120\u00e2\u0128": 17804, "mill": 17805, "\u0120varies": 17806, "\u0120Pakistani": 17807, "Dragon": 17808, "\u0120une": 17809, "\u0120horizon": 17810, "\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142": 17811, "\u0120provinces": 17812, "\u0120frankly": 17813, "\u0120enacted": 17814, "notes": 17815, "['": 17816, "\u0120192": 17817, "ocracy": 17818, "\u0120endorsement": 17819, "\u0120overtime": 17820, "True": 17821, "Lab": 17822, "licted": 17823, "\u0120DNC": 17824, "\u0120beats": 17825, "\u0120Jamie": 17826, "152": 17827, "\u0120INT": 17828, "Contact": 17829, "\u0120accounted": 17830, "hash": 17831, "\u0120Packers": 17832, "pires": 17833, "\u0120lesbian": 17834, "\u0120amendments": 17835, "\u0120hopeful": 17836, "\u0120Finland": 17837, "\u0120spotlight": 17838, "\u0120configured": 17839, "\u0120troubled": 17840, "\u0120gaze": 17841, "\u0120Calgary": 17842, "\u0120reliability": 17843, "\u0120insurg": 17844, "swer": 17845, "buy": 17846, "\u0120Skin": 17847, "\u0120pixels": 17848, "\u0120handgun": 17849, "\u0120paras": 17850, "\u0120categor": 17851, "\u0120EL": 17852, "\u0120Rex": 17853, "Indeed": 17854, "\u0120kinda": 17855, "\u0120conjunction": 17856, "\u0120Bryan": 17857, "\u0120Manufact": 17858, "yang": 17859, "Plus": 17860, "SQL": 17861, "ishment": 17862, "\u0120dominate": 17863, "\u0120nail": 17864, "\u0120oath": 17865, "\u0120erupt": 17866, "\u0120Fine": 17867, "itbart": 17868, "\u0120Chip": 17869, "\u0120Abd": 17870, "\u0120Nam": 17871, "\u0120buyer": 17872, "\u0120dissent": 17873, "Leaks": 17874, "Contin": 17875, "\u0120rider": 17876, "\u0120Someone": 17877, "\u0120illusion": 17878, "cin": 17879, "\u0120Boeing": 17880, "\u0120inadequ": 17881, "ovation": 17882, "iants": 17883, "\u0120rebuild": 17884, "450": 17885, "\u0120Destiny": 17886, "SW": 17887, "\u0120Till": 17888, "Hit": 17889, "iaz": 17890, "\u0120Bangl": 17891, "achers": 17892, "\u0120Reform": 17893, "\u0120segments": 17894, "\u0120systematic": 17895, "dc": 17896, "\u0120Conservatives": 17897, "\u0120portal": 17898, "hor": 17899, "\u0120Dragonbound": 17900, "\u0120dragged": 17901, "omo": 17902, "\u0120thee": 17903, "advert": 17904, "\u0120Reports": 17905, "\u0120Et": 17906, "\u0120barrels": 17907, "August": 17908, "\u0120comparisons": 17909, "\u0120hex": 17910, "\u0120anthrop": 17911, "\"[": 17912, "borough": 17913, "abi": 17914, "\u0120pictured": 17915, "playing": 17916, "\u0120Address": 17917, "\u0120Mirror": 17918, "Smith": 17919, "\u0120tires": 17920, "\u0120NPR": 17921, "AAAA": 17922, "\u0120classification": 17923, "\u0120Than": 17924, "\u0120Harm": 17925, "\u0120RA": 17926, "\u0120rejection": 17927, "mination": 17928, "\u0120ranged": 17929, "\u0120Falls": 17930, "DI": 17931, "Host": 17932, "\u00e3\u0124\u00b4": 17933, "\u0120Example": 17934, "listed": 17935, "thirds": 17936, "\u0120safegu": 17937, "brand": 17938, "\u0120probable": 17939, "Canada": 17940, "ITION": 17941, "\u0120Qaeda": 17942, "\u0120chick": 17943, "\u0120imports": 17944, "hit": 17945, "loc": 17946, "WW": 17947, "\u0120blew": 17948, "\u0120anytime": 17949, "\u0120wholes": 17950, "iked": 17951, "\u0120calculation": 17952, "create": 17953, "\u0120Ori": 17954, "\u0120upgraded": 17955, "\u0120appar": 17956, "utory": 17957, "\u0120Mol": 17958, "Brit": 17959, "\u0120Jong": 17960, "INAL": 17961, "\u0120Starting": 17962, "\u0120dice": 17963, "urtle": 17964, "\u0120relying": 17965, "closure": 17966, "\u0120profitable": 17967, "\u0120slaughter": 17968, "\u0120Manual": 17969, "caster": 17970, "\u0120\"$": 17971, "\u0120feather": 17972, "\u0120Simply": 17973, "ieves": 17974, "\u0120deterior": 17975, "\u0120PCI": 17976, "\u0120stamp": 17977, "\u0120flaws": 17978, "\u0120shade": 17979, "hammer": 17980, "\u0120passport": 17981, "\u0120conting": 17982, "amel": 17983, "\u0120observers": 17984, "\u0120neglect": 17985, "\u0120RB": 17986, "\u0120Brotherhood": 17987, "\u0120skeptical": 17988, "family": 17989, "usk": 17990, "\u0120emotionally": 17991, "\u00e2\u013b": 17992, "\u0120Beta": 17993, "asonable": 17994, "idity": 17995, "\u0120Mul": 17996, "\u0120kicking": 17997, "\u0120Carm": 17998, "ollah": 17999, "VERTIS": 18000, "\u0120Athen": 18001, "\u0120ladder": 18002, "\u0120Bullet": 18003, "\u00e5\u00a3": 18004, "0001": 18005, "\u0120Wildlife": 18006, "\u0120Mask": 18007, "\u0120Nan": 18008, "Rev": 18009, "\u0120unacceptable": 18010, "legal": 18011, "\u0120crowded": 18012, "agi": 18013, "\u0120Cox": 18014, "je": 18015, "\u0120morality": 18016, "\u0120fuels": 18017, "\u0120cables": 18018, "\u0120mankind": 18019, "\u0120Caribbean": 18020, "\u0120anchor": 18021, "\u0120byte": 18022, "\u0120Often": 18023, "\u0120Oz": 18024, "\u0120crafted": 18025, "\u0120historian": 18026, "\u0120Wu": 18027, "\u0120towers": 18028, "\u0120Citizens": 18029, "\u0120helm": 18030, "\u0120credentials": 18031, "\u0120singular": 18032, "\u0120Jesse": 18033, "\u0120tackles": 18034, "\u0120contempt": 18035, "\u0120afore": 18036, "\u0120Shadows": 18037, "\u0120nil": 18038, "\u0120urgent": 18039, "apple": 18040, "blood": 18041, "\u0120von": 18042, "\u0120offline": 18043, "\u0120breathe": 18044, "\u0120jumps": 18045, "\u0120irrelevant": 18046, "oxic": 18047, "omal": 18048, "important": 18049, "Jim": 18050, "\u0120gloves": 18051, "arming": 18052, "depth": 18053, "\u0120talents": 18054, "ookie": 18055, "\u0120SB": 18056, "\u0120palm": 18057, "uffs": 18058, "esta": 18059, "IGH": 18060, "\u0120canon": 18061, "\u0120Verizon": 18062, "\u0120Ple": 18063, "\u0120coupled": 18064, "velt": 18065, "\u0120fundraising": 18066, "\u0120Getting": 18067, "\u0120DLC": 18068, "\u0120mathematical": 18069, "\u0120HS": 18070, "\u0120Cardinals": 18071, "telling": 18072, "\u0120sponsors": 18073, "\u0120\u00cf": 18074, "\u0120Bulls": 18075, "option": 18076, "\u0120propose": 18077, "\u0120memorable": 18078, "\u0120embraced": 18079, "\u0120declining": 18080, "Health": 18081, "eda": 18082, "\u0120};": 18083, "\u0120spam": 18084, "mile": 18085, "\u0120pitcher": 18086, "\u0120Eight": 18087, "\u0120caring": 18088, "utic": 18089, "role": 18090, "\u0120airline": 18091, "ernandez": 18092, "\u0120Athlet": 18093, "\u0120certification": 18094, "uxe": 18095, "riger": 18096, "\u0120empir": 18097, "\u0120sensation": 18098, "\u0120dism": 18099, "\u0120bolt": 18100, "\u0120evolve": 18101, "House": 18102, "\u0120consultation": 18103, "\u0120Duty": 18104, "\u0120touches": 18105, "\u0120Nathan": 18106, "\u0120faint": 18107, "had": 18108, "\"(": 18109, "\u0120Consumer": 18110, "\u0120Extreme": 18111, "\u0120127": 18112, "\u0120Herm": 18113, "\u0120Sacrament": 18114, "izoph": 18115, "\u0120anxious": 18116, "ulously": 18117, "\u0120socially": 18118, "\u0120UTC": 18119, "\u0120solving": 18120, "\u0120Letter": 18121, "History": 18122, "educ": 18123, "Price": 18124, "));": 18125, "\u0120reload": 18126, "amic": 18127, "\u0120pork": 18128, "\u0120discourse": 18129, "\u0120tournaments": 18130, "airo": 18131, "\u0120Kur": 18132, "\u0120Costa": 18133, "\u0120violating": 18134, "\u0120interfere": 18135, "\u0120recreational": 18136, "uffle": 18137, "\u0120speeches": 18138, "\u0120needing": 18139, "\u0120remembers": 18140, "\u0120credited": 18141, "nia": 18142, "focused": 18143, "amera": 18144, "\u0120bru": 18145, "umbs": 18146, "\u0120Cuban": 18147, "\u0120preceding": 18148, "\u0120nonsense": 18149, "acial": 18150, "\u0120smartphones": 18151, "\u0120Stories": 18152, "Sports": 18153, "\u0120Emergency": 18154, "ouncing": 18155, "efined": 18156, "\u0120ber": 18157, "\u0120consulting": 18158, "\u0120masters": 18159, "heastern": 18160, ".\"[": 18161, "\u0120Running": 18162, "\u0120suscept": 18163, "\u0120Feng": 18164, "America": 18165, "prises": 18166, "stitial": 18167, "\u0120Weekly": 18168, "\u0120Greater": 18169, "modules": 18170, "ifter": 18171, "Graphics": 18172, "uler": 18173, "\u0120wholly": 18174, "\u0120suppress": 18175, "\u0120concealed": 18176, "\u0120happily": 18177, "\u0120accepts": 18178, "\u0120Enjoy": 18179, "\u0120rivers": 18180, "\u0120Except": 18181, "225": 18182, "\u0120NHS": 18183, "\u0120McConnell": 18184, "\u0120pussy": 18185, "ferred": 18186, "utable": 18187, "\u0120attain": 18188, "\u0120>=": 18189, "\u0120deposits": 18190, "rophic": 18191, "\u0120notorious": 18192, "\u0120Shaw": 18193, "ilitation": 18194, "\u0120epidemic": 18195, "allic": 18196, "\u0120smallest": 18197, "ovich": 18198, "\u0120accessories": 18199, "perties": 18200, "\u0120surplus": 18201, "\u0120Mech": 18202, "\u0120ambig": 18203, "\u0120Immigration": 18204, "\u0120chim": 18205, "eval": 18206, "\u0120practicing": 18207, "\u0120Mystery": 18208, "\u0120domains": 18209, "\u0120Silicon": 18210, "apps": 18211, "\u0120kilometers": 18212, "ea": 18213, "\u0120Smash": 18214, "\u0120warranty": 18215, "\u0120nost": 18216, "sil": 18217, "rev": 18218, "Jon": 18219, "\u0120Dublin": 18220, "\u0120tastes": 18221, "\u0120bout": 18222, "great": 18223, "error": 18224, "\u0120switches": 18225, "\u0120Bapt": 18226, "DO": 18227, "oki": 18228, "\u0120sourced": 18229, "produ": 18230, "\u0120attachment": 18231, "\u0120Issue": 18232, "\u0120Question": 18233, "Join": 18234, "\u0120fitted": 18235, "\u0120unlawful": 18236, "^^": 18237, "erek": 18238, "\u0120authentication": 18239, "\u0120stole": 18240, "\u0120accountability": 18241, "label": 18242, "Search": 18243, "\u0120albeit": 18244, "atican": 18245, "funded": 18246, "\u0120Adding": 18247, "\u0120IQ": 18248, "\u0120submar": 18249, "lit": 18250, "aque": 18251, "\u0120Learning": 18252, "\u0120integer": 18253, "Master": 18254, "\u0120Chrom": 18255, "\u0120premier": 18256, "Op": 18257, "\u0120Liu": 18258, "\u0120blessed": 18259, "\u0120Globe": 18260, "\u0120Response": 18261, "\u0120legitim": 18262, "\u0120Merkel": 18263, "\u0120disposal": 18264, "\u00c2\u00b4": 18265, "\u0120gauge": 18266, "peat": 18267, "\u0120induced": 18268, "\u0120questionable": 18269, "arthy": 18270, "\u0120Vit": 18271, "\u0120Feed": 18272, "Until": 18273, "Ut": 18274, "worthy": 18275, "RY": 18276, "\u0120Herald": 18277, "\u0120Hammer": 18278, "\u0120medal": 18279, "\u0120Rivers": 18280, "\u0120Hack": 18281, "\u0120clarify": 18282, "\u0120tracked": 18283, "\u0120autonomous": 18284, "\u0120tenant": 18285, "\u0120Qatar": 18286, "erie": 18287, "\u0120grim": 18288, "\u0120Monitor": 18289, "\u0120resistant": 18290, "\u0120Spec": 18291, "\u0120Wells": 18292, "NAS": 18293, "148": 18294, "\u0120miners": 18295, "iotics": 18296, "\u0120misses": 18297, "116": 18298, "gian": 18299, "git": 18300, "\u0120Eyes": 18301, "pres": 18302, "\u0120graduated": 18303, "\u0120angel": 18304, "\u0120synchron": 18305, "\u0120efficiently": 18306, "\u0120transmitted": 18307, "Harry": 18308, "\u0120globally": 18309, "ENCE": 18310, "\u0120Montana": 18311, "raged": 18312, "\u0120Prevention": 18313, "\u0120piss": 18314, "\u0120Ll": 18315, "\u0120shelf": 18316, "\u0120BJP": 18317, "\u0120Testament": 18318, "\u0120Late": 18319, "iker": 18320, "\u0120Happ": 18321, "\u0120Julian": 18322, "hall": 18323, "\u0120spont": 18324, "\u0120shutdown": 18325, "\u0120inconsistent": 18326, "\u0120subscribers": 18327, "\u0120skeleton": 18328, "\u0120Nebraska": 18329, "\u0120inspire": 18330, "\u0120Void": 18331, "Feed": 18332, "\u0120angles": 18333, "\u0120Springs": 18334, "\u0120benchmark": 18335, "\u0120vaccines": 18336, "izophren": 18337, "sexual": 18338, "uffed": 18339, "\u0120shine": 18340, "\u0120Kath": 18341, "\u0120gesture": 18342, "inea": 18343, "\u0120rip": 18344, "\u0120oppression": 18345, "\u0120conscience": 18346, "bt": 18347, "\u0120Lum": 18348, "\u0120incidence": 18349, "\u0120Fa": 18350, "wr": 18351, "\u0120mineral": 18352, "\u0120Spurs": 18353, "alky": 18354, "\u0120thunder": 18355, "\u0120opio": 18356, "Being": 18357, "\u0120Palm": 18358, "\u0120wasted": 18359, "\u0120lb": 18360, "iaries": 18361, "\u0120Initiative": 18362, "\u0120curric": 18363, "\u0120marker": 18364, "\u0120McL": 18365, "\u0120extensions": 18366, "\u0120Pv": 18367, "\u0120Arms": 18368, "\u0120offerings": 18369, "\u0120defenses": 18370, "\u0120vendor": 18371, "\u0120contradict": 18372, "\u0120Colin": 18373, "\u0120reddit": 18374, "\u0120peripher": 18375, "122": 18376, "\u0120sins": 18377, "Edit": 18378, "ICT": 18379, "Soft": 18380, "\u0120Shah": 18381, "\u0120administrator": 18382, "\u0120Trip": 18383, "\u0120pornography": 18384, "\u0120tuition": 18385, "inence": 18386, "\u0120Progress": 18387, "\u0120catalog": 18388, "\u0120suite": 18389, "\u0120hike": 18390, "\u0120reproductive": 18391, "engine": 18392, "\u0120drought": 18393, "\u0120Noah": 18394, "\u0120230": 18395, "\u0120dude": 18396, "\u0120relaxed": 18397, "\u0120partition": 18398, "\u0120participant": 18399, "\u0120telesc": 18400, "\u0120feas": 18401, "\u0120FF": 18402, "owner": 18403, "\u0120sweeping": 18404, "\u0120lenses": 18405, "\u0120matchup": 18406, "\u0120Repl": 18407, "ournals": 18408, "\u0120credible": 18409, "\u0120grandmother": 18410, "\u0120thermal": 18411, "\u0120subscribing": 18412, "\u0120identities": 18413, "colm": 18414, "UCT": 18415, "\u0120reluctant": 18416, "users": 18417, "\u0120Cort": 18418, "\u0120assisted": 18419, "OSS": 18420, "ATIONS": 18421, "ISH": 18422, "\u0120pharmaceutical": 18423, "icable": 18424, "adian": 18425, "\u0120Sonic": 18426, "\u0120Fury": 18427, "\u0120Mong": 18428, "AH": 18429, "\u0120Psychology": 18430, "\u0120phosph": 18431, "\u0120treats": 18432, "\u0143\u0136": 18433, "\u0120steadily": 18434, "\u0120Hello": 18435, "\u0120relates": 18436, "\u0120clue": 18437, "Expl": 18438, "auth": 18439, "\u0120revision": 18440, "\u0120eld": 18441, "osion": 18442, "\u0120bron": 18443, "144": 18444, "rikes": 18445, "\u0120mines": 18446, "\u0120blanket": 18447, "\u0120Fail": 18448, "eled": 18449, "\u0120Imagine": 18450, "\u0120Planned": 18451, "aic": 18452, "Request": 18453, "Mad": 18454, "\u0120Horse": 18455, "\u0120Eagle": 18456, "\u0120capac": 18457, "157": 18458, "\u0120ling": 18459, "\u0120Nice": 18460, "\u0120Parenthood": 18461, "minster": 18462, "ogs": 18463, "ensitive": 18464, "Nothing": 18465, "\u0120carn": 18466, "Fin": 18467, "\u0120PE": 18468, "\u0120rifles": 18469, "\u0120LP": 18470, "Sand": 18471, "\u0120guiActive": 18472, "\u0120tourist": 18473, "CNN": 18474, "\u0120unveiled": 18475, "\u0120predecessor": 18476, "}{": 18477, "uber": 18478, "\u0120offshore": 18479, "\u0120optical": 18480, "\u0120Rot": 18481, "\u0120Pearl": 18482, "eton": 18483, "\u0120stared": 18484, "\u0120farther": 18485, "atility": 18486, "contin": 18487, "\u0120Gy": 18488, "\u0120Foster": 18489, "\u0120Coc": 18490, "rients": 18491, "\u0120designing": 18492, "\u0120Economy": 18493, "ONG": 18494, "Women": 18495, "\u0120Nancy": 18496, "erver": 18497, "\u0120mascul": 18498, "\u0120casualties": 18499, "\u0120225": 18500, "\u0120Sullivan": 18501, "\u0120Choice": 18502, "\u0120aster": 18503, "ws": 18504, "\u0120hotels": 18505, "\u0120considerations": 18506, "\u0120couch": 18507, "\u0120Strip": 18508, "\u0120Gn": 18509, "\u0120manipulate": 18510, "lied": 18511, "\u0120synthetic": 18512, "\u0120assaulted": 18513, "\u0120offenses": 18514, "\u0120Drake": 18515, "\u0120impe": 18516, "October": 18517, "\u0120Heritage": 18518, "hl": 18519, "\u0120Blair": 18520, "Unlike": 18521, "\u0120grief": 18522, "\u0120450": 18523, "\u0120opted": 18524, "\u0120resignation": 18525, "ilo": 18526, "\u0120verse": 18527, "\u0120Tomb": 18528, "\u0120upt": 18529, "\u0120aired": 18530, "\u0120Hook": 18531, "\u0120MLB": 18532, "\u0120assumes": 18533, "outed": 18534, "\u0120Vers": 18535, "\u0120inferior": 18536, "\u0120bundle": 18537, "\u0120DNS": 18538, "ographer": 18539, "\u0120multip": 18540, "\u0120Souls": 18541, "\u0120illustrated": 18542, "\u0120tactic": 18543, "\u0120dressing": 18544, "\u0120duo": 18545, "Conf": 18546, "\u0120relent": 18547, "\u0120cant": 18548, "\u0120scarce": 18549, "\u0120candy": 18550, "\u0120CF": 18551, "\u0120affiliated": 18552, "\u0120sprint": 18553, "ylan": 18554, "\u0120Garcia": 18555, "\u0120junk": 18556, "Print": 18557, "exec": 18558, "Crit": 18559, "\u0120portrait": 18560, "iries": 18561, "\u0120OFF": 18562, "\u0120disputes": 18563, "WR": 18564, "Love": 18565, "\u00e3\u0123\u0126": 18566, "\u0120Reyn": 18567, "\u0120hipp": 18568, "opath": 18569, "\u0120floors": 18570, "\u0120Feel": 18571, "\u0120worries": 18572, "\u0120settlements": 18573, "\u0120Pos": 18574, "\u0120mosque": 18575, "\u0120finals": 18576, "\u0120crushed": 18577, "\u0120Probably": 18578, "\u0120Bot": 18579, "\u0120Mans": 18580, "\u0120Period": 18581, "\u0120sovereignty": 18582, "\u0120seller": 18583, "\u0120apost": 18584, "\u0120amateur": 18585, "\u0120dorm": 18586, "\u0120consuming": 18587, "\u0120armour": 18588, "\u0120Roose": 18589, "\u0120intensive": 18590, "\u0120eliminating": 18591, "\u0120Sunni": 18592, "\u0120Aleppo": 18593, "jin": 18594, "\u0120advise": 18595, "pal": 18596, "\u0120Halo": 18597, "\u0120descent": 18598, "\u0120simpler": 18599, "\u0120booth": 18600, "STR": 18601, "Later": 18602, "\u0120Cave": 18603, "===": 18604, "\u0120mol": 18605, "\u0120fist": 18606, "\u0120shotgun": 18607, "supp": 18608, "\u0120robbery": 18609, "Effect": 18610, "\u0120obscure": 18611, "\u0120Professional": 18612, "\u0120embassy": 18613, "\u0120militant": 18614, "\u0120incarcer": 18615, "\u0120generates": 18616, "\u0120launches": 18617, "\u0120administrators": 18618, "\u0120shaft": 18619, "\u0120circular": 18620, "\u0120freshman": 18621, "\u0120Wes": 18622, "\u0120Joel": 18623, "\u0120Drew": 18624, "\u0120Duncan": 18625, "\u0120Apparently": 18626, "sight": 18627, "\u0120Internal": 18628, "\u0120Individual": 18629, "\u0120FE": 18630, "\u0120bore": 18631, "\u0120Mt": 18632, "\u0120broadly": 18633, "\u0120Options": 18634, "ountain": 18635, "ipes": 18636, "\u0120Videos": 18637, "204": 18638, "\u0120hills": 18639, "\u0120simulation": 18640, "\u0120disappointment": 18641, "itan": 18642, "\u0120Laboratory": 18643, "\u0120upward": 18644, "\u0120boundary": 18645, "\u0120darker": 18646, "hart": 18647, "\u0120dominance": 18648, "Cong": 18649, "\u0120Oracle": 18650, "\u0120Lords": 18651, "\u0120scholarship": 18652, "\u0120Vincent": 18653, "ede": 18654, "\u0120Rah": 18655, "\u0120encourages": 18656, "rov": 18657, "\u0120quo": 18658, "\u0120premise": 18659, "\u0120Crisis": 18660, "\u0120Holocaust": 18661, "\u0120rhythm": 18662, "\u0120metric": 18663, "club": 18664, "\u0120transported": 18665, "\u0120nod": 18666, "\u0120Pist": 18667, "\u0120ancestors": 18668, "\u0120Freder": 18669, "thumbnails": 18670, "\u0120CE": 18671, "OND": 18672, "Phil": 18673, "venge": 18674, "\u0120Products": 18675, "castle": 18676, "\u0120qualifying": 18677, "\u0120Karen": 18678, "VERTISEMENT": 18679, "\u0120mighty": 18680, "\u0120explanations": 18681, "\u0120fixing": 18682, "Di": 18683, "\u0120declaring": 18684, "\u0120anonymity": 18685, "\u0120juven": 18686, "\u0120Nord": 18687, "\u0120Doom": 18688, "\u0120Actually": 18689, "Ok": 18690, "phis": 18691, "\u0120Desert": 18692, "\u0120116": 18693, "IK": 18694, "\u0120FM": 18695, "\u0120incomes": 18696, "VEL": 18697, "okers": 18698, "\u0120pecul": 18699, "\u0120lightweight": 18700, "gue": 18701, "\u0120accent": 18702, "\u0120increment": 18703, "\u0120Chan": 18704, "\u0120complaining": 18705, "\u0120Baghd": 18706, "\u0120midfielder": 18707, "\u0120overhaul": 18708, "Process": 18709, "\u0120Hollow": 18710, "\u0120Titans": 18711, "Small": 18712, "manuel": 18713, "\u0120Unity": 18714, "\u0120Events": 18715, "Sty": 18716, "\u0120disproportion": 18717, "nesty": 18718, "enes": 18719, "\u0120Cod": 18720, "\u0120demonstrations": 18721, "\u0120Crimson": 18722, "\u0120OH": 18723, "\u0120enrolled": 18724, "\u0120cel": 18725, "\u0120Brett": 18726, "\u0120aide": 18727, "\u0120heels": 18728, "\u0120broadband": 18729, "\u0120marking": 18730, "\u0120wizard": 18731, "\u0120NJ": 18732, "\u0120Chiefs": 18733, "\u0120ingredient": 18734, "\u0120dug": 18735, "\u0120Shut": 18736, "urchase": 18737, "endor": 18738, "\u0120farmer": 18739, "\u0120Goldman": 18740, "129": 18741, "155": 18742, "Order": 18743, "\u0120lion": 18744, "iably": 18745, "\u0120stain": 18746, "array": 18747, "ilitary": 18748, "\u0120FAQ": 18749, "\u0120exploded": 18750, "\u0120McCarthy": 18751, "\u0120Tweet": 18752, "\u0120Greens": 18753, "eking": 18754, "ln": 18755, "ensen": 18756, "\u0120motorcycle": 18757, "\u0120particle": 18758, "\u0120cholesterol": 18759, "Bron": 18760, "\u0120stair": 18761, "\u0120oxid": 18762, "\u0120desirable": 18763, "ibles": 18764, "\u0120theor": 18765, "forcing": 18766, "\u0120promotional": 18767, "ovo": 18768, "boot": 18769, "\u0120Bonus": 18770, "rawling": 18771, "\u0120shortage": 18772, "\u0120Psy": 18773, "\u0120recruited": 18774, "\u0120infants": 18775, "\u0120testosterone": 18776, "\u0120deduct": 18777, "\u0120distinctive": 18778, "\u0120firmware": 18779, "built": 18780, "145": 18781, "\u0120explored": 18782, "\u0120factions": 18783, "\u0120vide": 18784, "\u0120tattoo": 18785, "\u0120financially": 18786, "\u0120fatigue": 18787, "\u0120proceeding": 18788, "constitutional": 18789, "\u0120miser": 18790, "\u0120chairs": 18791, "gging": 18792, "ipple": 18793, "\u0120dent": 18794, "\u0120disreg": 18795, "\u00e7\u0136": 18796, "stant": 18797, "llo": 18798, "bps": 18799, "akening": 18800, "\u0120abnormal": 18801, "\u0120ERA": 18802, "\u00e5\u00a3\u00ab": 18803, "\u0120HBO": 18804, "\u0120MAR": 18805, "\u0120concess": 18806, "\u0120servant": 18807, "\u0120aspir": 18808, "lav": 18809, "\u0120Panel": 18810, "amo": 18811, "\u0120precip": 18812, "\u0120recordings": 18813, "\u0120proceeded": 18814, "\u0120colony": 18815, "\u0120Tang": 18816, "ablo": 18817, "\u0120stripped": 18818, "Left": 18819, "too": 18820, "\u0120potatoes": 18821, "\u0120finest": 18822, "%).": 18823, "\u0120crap": 18824, "\u0120Zach": 18825, "abases": 18826, "\u0120Goth": 18827, "\u0120billionaire": 18828, "wolf": 18829, "\u0120sanction": 18830, "SK": 18831, "\u0120logged": 18832, "Po": 18833, "eyed": 18834, "unal": 18835, "\u0120cricket": 18836, "\u0120armies": 18837, "\u0120uncovered": 18838, "Cloud": 18839, "\u00c3\u00b3n": 18840, "\u0120rebounds": 18841, "\u0120mes": 18842, "Oper": 18843, "Pac": 18844, "\u0120nationally": 18845, "\u0120inserted": 18846, "pict": 18847, "\u0120governance": 18848, "\u00d0\u00b8": 18849, "\u0120privileges": 18850, "GET": 18851, "\u0120favorites": 18852, "imity": 18853, "\u0120lover": 18854, "them": 18855, "empl": 18856, "\u0120gorgeous": 18857, "Ann": 18858, "\u0120slipped": 18859, "\u0120veto": 18860, "Bob": 18861, "\u0120slim": 18862, "ucc": 18863, "\u0120Fame": 18864, "uddenly": 18865, "\u0120denies": 18866, "\u0120Maur": 18867, "\u0120distances": 18868, "\u0120wanna": 18869, "tar": 18870, "\u0120SER": 18871, "\u0120\u00e2\u012a": 18872, "\u0120lemon": 18873, "athetic": 18874, "\u0120literal": 18875, "\u0120distinguished": 18876, "\u0120answering": 18877, "GI": 18878, "\u0120religions": 18879, "\u0120Philos": 18880, "\u0120Lay": 18881, "\u0120compos": 18882, "irements": 18883, "\u0120Kos": 18884, "inez": 18885, "rolling": 18886, "\u0120youngest": 18887, "andise": 18888, "\u0120Born": 18889, "\u0120altar": 18890, "amina": 18891, "\u0120Boot": 18892, "voc": 18893, "\u0120digging": 18894, "\u0120pressures": 18895, "\u0120len": 18896, "264": 18897, "\u0120assassination": 18898, "\u0120Birmingham": 18899, "\u0120Myth": 18900, "\u0120sovereign": 18901, "\u0120Artist": 18902, "\u0120Photograph": 18903, "\u0120depicted": 18904, "\u0120dispens": 18905, "orthy": 18906, "\u0120ambul": 18907, "integ": 18908, "\u0120Cele": 18909, "\u0120Tibet": 18910, "\u0120hierarchy": 18911, "\u0120cu": 18912, "\u0120preseason": 18913, "\u0120Peterson": 18914, "\u0120colours": 18915, "\u0120worrying": 18916, "\u0120backers": 18917, "\u0120Palmer": 18918, "\u0120\u00ce\u00bc": 18919, "\u0120contributor": 18920, "\u0120hearings": 18921, "\u0120urine": 18922, "\u0120\u00d9": 18923, "ourgeois": 18924, "Similar": 18925, "\u0120Zimmer": 18926, "something": 18927, "\u0120USC": 18928, "\u0120strengths": 18929, "\u0120FI": 18930, "\u0120logging": 18931, "Asked": 18932, "\u0120Thai": 18933, "inqu": 18934, "\u0120Walt": 18935, "\u0120crews": 18936, "itism": 18937, "301": 18938, "\u0120sharply": 18939, "umed": 18940, "\u0120redirect": 18941, "rators": 18942, "Inf": 18943, "\u0120Weapons": 18944, "\u0120teasp": 18945, "1999": 18946, "Live": 18947, "\u0120Especially": 18948, "\u0120Ster": 18949, "\u0120Veterans": 18950, "\u0120intro": 18951, "otherapy": 18952, "\u0120malware": 18953, "\u0120breeding": 18954, "\u0120molecular": 18955, "\u0120Route": 18956, "\u0120Comment": 18957, "ochem": 18958, "\u0120ain": 18959, "Season": 18960, "\u0120linebacker": 18961, "\u00c4\u00ab": 18962, "\u0120Economics": 18963, "esar": 18964, "\u0120Lives": 18965, "\u0120Emma": 18966, "\u0120kin": 18967, "\u0120Territ": 18968, "\u0120planted": 18969, "oton": 18970, "\u0120Butter": 18971, "\u0120Spons": 18972, "PER": 18973, "\u0120dungeon": 18974, "\u0120symbolic": 18975, "\u0120filmed": 18976, "\u0120diets": 18977, "\u0120concludes": 18978, "\u0120certainty": 18979, "\u0120Format": 18980, "\u0120strangers": 18981, "format": 18982, "\u0120Phase": 18983, "\u0120copied": 18984, "\u0120metres": 18985, "lda": 18986, "\u0120Users": 18987, "\u0120deliberate": 18988, "\u0120washed": 18989, "\u0120Lance": 18990, "imation": 18991, "\u0120improper": 18992, "\u0120Genesis": 18993, "ickr": 18994, "\u0120Kush": 18995, "\u0120realise": 18996, "\u0120embarrassing": 18997, "alking": 18998, "bucks": 18999, "\u0120verified": 19000, "\u0120outline": 19001, "years": 19002, "\u0120Income": 19003, "202": 19004, "\u0120zombies": 19005, "Final": 19006, "\u0120Millenn": 19007, "\u0120modifications": 19008, "\u0120Vision": 19009, "\u0120Moses": 19010, "verb": 19011, "iterranean": 19012, "\u0120Jet": 19013, "\u0120naval": 19014, "\u0120Agg": 19015, "\u0120url": 19016, "\u0120victories": 19017, "\u0120nonetheless": 19018, "\u0120injust": 19019, "\u0120Fact": 19020, "\u00e7\u013c": 19021, "\u0120insufficient": 19022, "review": 19023, "facebook": 19024, "\u0120negotiating": 19025, "\u0120guarantees": 19026, "imen": 19027, "utenberg": 19028, "\u0120gambling": 19029, "\u0120congr": 19030, "Loading": 19031, "\u0120nevertheless": 19032, "\u0120presidents": 19033, "\u0120Industrial": 19034, "\u0120118": 19035, "\u0120poured": 19036, "\u0120Tory": 19037, "\u0120175": 19038, "\u0120:=": 19039, "Scott": 19040, "angered": 19041, "Tok": 19042, "\u0120organizers": 19043, "Mat": 19044, "\u0120Growth": 19045, "\u0120adul": 19046, "\u0120ensures": 19047, "\u0120117": 19048, "\u00e9\u00be\u012f\u00e5": 19049, "\u0120massacre": 19050, "\u0120grades": 19051, "before": 19052, "ADVERTISEMENT": 19053, "\u0120Slow": 19054, "\u0120MMA": 19055, "\u00e2\u0122\u0136\"": 19056, "\u0120Vatican": 19057, "Qaeda": 19058, "\u0120owe": 19059, "6666": 19060, "\u0120Sorry": 19061, "\u0120Grass": 19062, "\u0120backgrounds": 19063, "\u0120exhausted": 19064, "\u0120clan": 19065, "\u0120compromised": 19066, "\u0120Elf": 19067, "\u0120Isaac": 19068, "enson": 19069, "Invest": 19070, "IFA": 19071, "\u0120interrupted": 19072, "\u00e3\u0125\u012b\u00e3\u0125\u00a9": 19073, "\u0120twisted": 19074, "\u0120Dragons": 19075, "Mode": 19076, "\u0120Kremlin": 19077, "\u0120fertil": 19078, "heres": 19079, "phan": 19080, "\u0120Node": 19081, "fed": 19082, "\u0120Orc": 19083, "\u0120unwilling": 19084, "Cent": 19085, "\u0120priorit": 19086, "\u0120graduates": 19087, "\u0120subjective": 19088, "\u0120issuing": 19089, "\u0120Lt": 19090, "\u0120viewer": 19091, "\u0120woke": 19092, "Thus": 19093, "brook": 19094, "\u0120depressed": 19095, "\u0120bracket": 19096, "\u0120Gor": 19097, "\u0120Fighting": 19098, "\u0120striker": 19099, "Report": 19100, "\u0120Portugal": 19101, "\u0120neo": 19102, "wed": 19103, "199": 19104, "\u0120fleeing": 19105, "shadow": 19106, "identified": 19107, "USE": 19108, "Steam": 19109, "\u0120stretched": 19110, "\u0120revelations": 19111, "arted": 19112, "\u0120Dw": 19113, "\u0120alignment": 19114, "eston": 19115, "\u0120Jared": 19116, "Sep": 19117, "\u0120blogs": 19118, "update": 19119, "gom": 19120, "risk": 19121, "\u0120clash": 19122, "\u0120Hour": 19123, "\u0120runtime": 19124, "\u0120unwanted": 19125, "\u0120scam": 19126, "\u0120rack": 19127, "\u0120enlight": 19128, "onest": 19129, "\u0120Ferr": 19130, "\u0120convictions": 19131, "\u0120piano": 19132, "\u0120circulation": 19133, "\u0120Welcome": 19134, "\u0120backlash": 19135, "\u0120Wade": 19136, "\u0120receivers": 19137, "otive": 19138, "Jeff": 19139, "\u0120networking": 19140, "\u0120Prep": 19141, "\u0120Explorer": 19142, "\u0120lecture": 19143, "\u0120uploaded": 19144, "\u0120Meat": 19145, "BLE": 19146, "\u0120Nazis": 19147, "\u0120Synd": 19148, "stud": 19149, "roots": 19150, "rians": 19151, "\u0120portrayed": 19152, "\u0120??": 19153, "\u0120Buddha": 19154, "sun": 19155, "Robert": 19156, "\u0120Complex": 19157, "\u0120oversee": 19158, "\u0120stealth": 19159, "Title": 19160, "\u0120Jobs": 19161, "\u0120Kum": 19162, "\u0120appreciation": 19163, "\u0120MOD": 19164, "\u0120basics": 19165, "\u0120clips": 19166, "\u0120nursing": 19167, "\u0120proposition": 19168, "\u0120realised": 19169, "\u0120NYC": 19170, "\u0120allocated": 19171, "rium": 19172, "aran": 19173, "\u0120Production": 19174, "\u0120Vote": 19175, "\u0120smugg": 19176, "\u0120hunter": 19177, "azer": 19178, "\u0120Changes": 19179, "\u0120fluct": 19180, "yon": 19181, "Array": 19182, "\u0120kits": 19183, "Water": 19184, "\u0120uncommon": 19185, "\u0120resting": 19186, "ells": 19187, "would": 19188, "\u0120pursued": 19189, "\u0120assertion": 19190, "ometown": 19191, "\u0120Mosul": 19192, "\u0120Platform": 19193, "iolet": 19194, "\u0120shareholders": 19195, "\u0120trails": 19196, "Pay": 19197, "\u0120Enforcement": 19198, "types": 19199, "\u0120Anonymous": 19200, "\u0120satisfying": 19201, "ilogy": 19202, "\u0120('": 19203, "wave": 19204, "city": 19205, "Steve": 19206, "\u0120confrontation": 19207, "\u0120Eld": 19208, "Capt": 19209, "ahan": 19210, "htm": 19211, "\u0120Ctrl": 19212, "ONS": 19213, "230": 19214, "ifa": 19215, "holding": 19216, "\u0120delicate": 19217, "\u0120jaw": 19218, "\u0120Going": 19219, "orum": 19220, "Sal": 19221, "\u0120dull": 19222, "\u0120Beth": 19223, "\u0120prisons": 19224, "\u0120ego": 19225, "\u0120Elsa": 19226, "avorite": 19227, "\u0120Gang": 19228, "\u0120Nuclear": 19229, "\u0120spider": 19230, "atsu": 19231, "\u0120sampling": 19232, "\u0120absorbed": 19233, "\u0120Pharm": 19234, "ieth": 19235, "\u0120bucket": 19236, "\u0120Recomm": 19237, "OF": 19238, "\u0120Factory": 19239, "ANCE": 19240, "\u0120bacter": 19241, "Has": 19242, "\u0120Observ": 19243, "121": 19244, "\u0120premiere": 19245, "Develop": 19246, "\u0120currencies": 19247, "Cast": 19248, "\u0120accompanying": 19249, "\u0120Nashville": 19250, "\u0120fatty": 19251, "\u0120Brend": 19252, "\u0120locks": 19253, "\u0120centered": 19254, "\u0120UT": 19255, "aughs": 19256, "orie": 19257, "\u0120Affordable": 19258, "vance": 19259, "DL": 19260, "emet": 19261, "\u0120throne": 19262, "\u0120Bluetooth": 19263, "\u0120naming": 19264, "ifts": 19265, "ADE": 19266, "\u0120corrected": 19267, "\u0120promptly": 19268, "\u0120STR": 19269, "\u0120genome": 19270, "\u0120cope": 19271, "\u0120valley": 19272, "\u0120rounded": 19273, "\u0120Kend": 19274, "alion": 19275, "pers": 19276, "\u0120tourism": 19277, "\u0120stark": 19278, "vl": 19279, "\u0120blowing": 19280, "\u0120Schedule": 19281, "std": 19282, "\u0120unhappy": 19283, "\u0120litigation": 19284, "cedes": 19285, "\u0120android": 19286, "\u0120integral": 19287, "erers": 19288, "uded": 19289, "tax": 19290, "\u0120reiter": 19291, "\u0120Motors": 19292, "ociated": 19293, "\u0120wonders": 19294, "\u0120Apost": 19295, "ucking": 19296, "\u0120Roosevelt": 19297, "fram": 19298, "\u0120yields": 19299, "\u0120constitutes": 19300, "awk": 19301, "Interest": 19302, "\u0120interim": 19303, "\u0120breakthrough": 19304, "\u0120Cher": 19305, "\u0120prosec": 19306, "\u0120Dj": 19307, "\u0120MT": 19308, "Resp": 19309, "\u0120PT": 19310, "\u0120sperm": 19311, "edit": 19312, "BT": 19313, "Linux": 19314, "country": 19315, "league": 19316, "\u0120dick": 19317, "\u0120oct": 19318, "\u0120inserting": 19319, "\u0120scra": 19320, "\u0120Brewing": 19321, "\u01201966": 19322, "\u0120runners": 19323, "\u0120plun": 19324, "idy": 19325, "\u0120Dian": 19326, "\u0120dysfunction": 19327, "\u0120exclusion": 19328, "\u0120disgr": 19329, "\u0120incorporate": 19330, "\u0120reconc": 19331, "\u0120nominated": 19332, "\u0120Archer": 19333, "draw": 19334, "achelor": 19335, "\u0120writings": 19336, "\u0120shallow": 19337, "\u0120hast": 19338, "\u0120BMW": 19339, "\u0120RS": 19340, "\u0120thigh": 19341, "\u01201963": 19342, "\u0120lamb": 19343, "\u0120favored": 19344, "agle": 19345, "\u0120cooler": 19346, "\u0120Hours": 19347, "\u0120GU": 19348, "\u0120Origin": 19349, "\u0120glimpse": 19350, "--------------------": 19351, "Lim": 19352, "\u0120cheek": 19353, "\u0120jealous": 19354, "-'": 19355, "\u0120harness": 19356, "\u0120Poison": 19357, "\u0120disabilities": 19358, "neapolis": 19359, "\u0120outlook": 19360, "\u0120notify": 19361, "\u0120Indianapolis": 19362, "\u0120abrupt": 19363, "nsic": 19364, "\u0120encrypted": 19365, "\u0120forfe": 19366, "reath": 19367, "\u0120rabb": 19368, "\u0120foundations": 19369, "\u0120compliment": 19370, "\u0120Interview": 19371, "\u0120Swe": 19372, "\u0120adolesc": 19373, "\u0120monitors": 19374, "\u0120Sacramento": 19375, "\u0120timely": 19376, "\u0120contempl": 19377, "\u0120positioned": 19378, "\u0120posters": 19379, "phies": 19380, "iovascular": 19381, "void": 19382, "\u0120Fifth": 19383, "\u0120investigative": 19384, "OUN": 19385, "\u0120integrate": 19386, "\u0120INC": 19387, "isha": 19388, "iblings": 19389, "\u0120Request": 19390, "\u0120Rodriguez": 19391, "\u0120slides": 19392, "\u0120DX": 19393, "\u0120feminism": 19394, "\u0120datas": 19395, "\u0120bend": 19396, "irus": 19397, "\u0120Nigeria": 19398, "Fox": 19399, "Change": 19400, "\u0120airplane": 19401, "\u0120Laden": 19402, "\u0120publicity": 19403, "ixty": 19404, "\u0120commitments": 19405, "\u0120aggregate": 19406, "\u0120displaying": 19407, "\u0120Arrow": 19408, "\u0120122": 19409, "\u0120respects": 19410, "android": 19411, "six": 19412, "\u0120Sha": 19413, "\u0120restoration": 19414, ")\\": 19415, "WS": 19416, "oys": 19417, "\u0120illustrate": 19418, "without": 19419, "126": 19420, "\u0120\u00e2\u0136\u0124": 19421, "\u0120pickup": 19422, "nels": 19423, "\u0120....": 19424, "food": 19425, "\u0120Fen": 19426, ")?": 19427, "\u0120phenomena": 19428, "\u0120companions": 19429, "\u0120Write": 19430, "\u0120spill": 19431, "\u0120bridges": 19432, "\u0120Updated": 19433, "\u0120Fo": 19434, "\u0120insects": 19435, "ASHINGTON": 19436, "\u0120scare": 19437, "iltr": 19438, "\u0120Zhang": 19439, "\u0120severity": 19440, "\u0120indul": 19441, "149": 19442, "\u0120Coffee": 19443, "\u0120norms": 19444, "\u0120pulse": 19445, "\u0120FT": 19446, "\u0120horrific": 19447, "\u0120Destroy": 19448, "\u0120JSON": 19449, "\u0120olive": 19450, "\u0120discusses": 19451, "Rest": 19452, "Elect": 19453, "\u0120Winn": 19454, "\u0120Surviv": 19455, "\u0120Hait": 19456, "Sure": 19457, "oped": 19458, "\u0120rooted": 19459, "\u0120Ske": 19460, "\u0120Bronze": 19461, "\u0120lol": 19462, "Default": 19463, "\u0120commodity": 19464, "redited": 19465, "\u0120libertarian": 19466, "\u0120forbidden": 19467, "\u0120gran": 19468, "\u00e0\u00a8": 19469, "\u0120lag": 19470, "enz": 19471, "drive": 19472, "\u0120mathematics": 19473, "\u0120wires": 19474, "\u0120critically": 19475, "\u0120carbohyd": 19476, "\u0120Chancellor": 19477, "\u0120Eddie": 19478, "\u0120banning": 19479, "\u0120Fri": 19480, "\u0120complications": 19481, "etric": 19482, "\u0120Bangladesh": 19483, "\u0120bandwidth": 19484, "Stop": 19485, "\u0120Originally": 19486, "\u0120halfway": 19487, "ynasty": 19488, "shine": 19489, "\u0120tales": 19490, "rities": 19491, "avier": 19492, "\u0120spinning": 19493, "\u0120WHO": 19494, "\u0120neighbourhood": 19495, "bach": 19496, "\u0120commerce": 19497, "\u0120Sle": 19498, "BU": 19499, "\u0120entrepreneur": 19500, "\u0120peculiar": 19501, "\u0120Comments": 19502, "fre": 19503, "320": 19504, "ICS": 19505, "\u0120imagery": 19506, "\u0120Canon": 19507, "\u0120Electronic": 19508, "short": 19509, "((": 19510, "Dig": 19511, "\u0120commem": 19512, "uced": 19513, "\u0120inclined": 19514, "\u0120Summon": 19515, "\u0120cliff": 19516, "\u0120Mediterranean": 19517, "\u0120poetry": 19518, "\u0120prosperity": 19519, "\u0120Rece": 19520, "\u0120pills": 19521, "member": 19522, "\u0120finale": 19523, "unc": 19524, "\u0120Gig": 19525, "\u00e4\u00bd": 19526, "\u0120lod": 19527, "\u0120backward": 19528, "-+": 19529, "\u0120Forward": 19530, "\u0120thri": 19531, "sure": 19532, "\u0120soap": 19533, "\u0120FX": 19534, "RES": 19535, "\u0120Sexual": 19536, "oulos": 19537, "\u0120foolish": 19538, "\u0120righteous": 19539, "\u0120coff": 19540, "terrorism": 19541, "ustain": 19542, "oter": 19543, "\u0120abuses": 19544, "next": 19545, "\u0120abusive": 19546, "\u0120thereafter": 19547, "\u0120prohibition": 19548, "\u0120SUP": 19549, "\u0120dip": 19550, "\u0120ripped": 19551, "\u0120inherited": 19552, "\u0120bats": 19553, "stru": 19554, "GT": 19555, "\u0120flawed": 19556, "phabet": 19557, "\u0120fog": 19558, "doors": 19559, "\u0120imaging": 19560, "\u0120digits": 19561, "\u0120Hungary": 19562, "\u0120arrog": 19563, "\u0120teachings": 19564, "\u0120protocols": 19565, "\u0120Banks": 19566, "\u00e0\u00b8": 19567, "pound": 19568, "\u0120Curt": 19569, ".\")": 19570, "./": 19571, "\u0120exemption": 19572, "endix": 19573, "\u0120Mull": 19574, "\u0120improves": 19575, "\u0120Gamer": 19576, "dimensional": 19577, "Icon": 19578, "\u0120Margaret": 19579, "Status": 19580, "dates": 19581, "\u0120intends": 19582, "\u0120depict": 19583, "\u0120parked": 19584, "Joe": 19585, "\u0120Marines": 19586, "chnology": 19587, "!).": 19588, "\u0120judged": 19589, "\u0120weights": 19590, "Ray": 19591, "\u0120apartments": 19592, "hester": 19593, "\u0120reinforce": 19594, "\u0120offender": 19595, "occup": 19596, "\u0120sore": 19597, "ept": 19598, "\u0120PHP": 19599, "\u0120Brow": 19600, "\u0120authorization": 19601, "\u0120Risk": 19602, "\u0120Delaware": 19603, "\u0120QU": 19604, "\u0120notifications": 19605, "\u0120sunlight": 19606, "\u0120exclude": 19607, "dat": 19608, "\u0120mesh": 19609, "\u0120Sudan": 19610, "\u0120belonged": 19611, "\u0120subway": 19612, "\u0120noon": 19613, "\u0120Interior": 19614, "olics": 19615, "\u0120Lakers": 19616, "\u0120coding": 19617, "Disclaimer": 19618, "Calif": 19619, "Old": 19620, "\u0120disl": 19621, "?????": 19622, "\u0120confirms": 19623, "\u0120recruitment": 19624, "\u0120homicide": 19625, "Consider": 19626, "\u0120Jeffrey": 19627, "fty": 19628, "};": 19629, "\u0120objection": 19630, "doing": 19631, "\u0120Leo": 19632, "Want": 19633, "\u0120glow": 19634, "\u0120Clarke": 19635, "\u0120Norman": 19636, "\u0120verification": 19637, "\u0120packet": 19638, "\u0120Formula": 19639, "\u0120plag": 19640, "esville": 19641, "\u0120shouting": 19642, "\u0120ov": 19643, "\u0120REC": 19644, "\u0120Bub": 19645, "\u0120ninth": 19646, "\u0120energ": 19647, "\u0120validity": 19648, "\u0120ups": 19649, "jack": 19650, "\u0120neighboring": 19651, "\u0120Nec": 19652, "eworks": 19653, "\u0120Hab": 19654, "arez": 19655, "\u0120spine": 19656, "\u0120eventual": 19657, "\u0120Leaders": 19658, "\u0120Carn": 19659, "\u0120probation": 19660, "\u0120romance": 19661, "msg": 19662, "\u0120Mechanical": 19663, "ERY": 19664, "Rock": 19665, "\u0120partisan": 19666, "Node": 19667, "assets": 19668, "minent": 19669, "\u0120foreigners": 19670, "\u0120testify": 19671, "\u0120Usually": 19672, "lords": 19673, "\u0120Gren": 19674, "\u0120Powell": 19675, "BIL": 19676, "\u0120sr": 19677, "\u0120addict": 19678, "\u0120shells": 19679, "\u0120sigh": 19680, "\u0120Yale": 19681, "ternity": 19682, "\u0120750": 19683, "EU": 19684, "\u0120Rifle": 19685, "\u0120patron": 19686, "ema": 19687, "\u0120Bannon": 19688, "anity": 19689, "\u0120tropical": 19690, "\u0120VII": 19691, "cross": 19692, "Everything": 19693, "\u0120ISO": 19694, "\u0120humble": 19695, "assing": 19696, "\u0120FIG": 19697, "\u0120updating": 19698, "yson": 19699, "\u0120calcium": 19700, "\u0120competent": 19701, "\u0120steering": 19702, "Prot": 19703, "\u0120SY": 19704, "\u0120Finals": 19705, "\u0120Rug": 19706, "159": 19707, "137": 19708, "\u0120Golf": 19709, "\u0120126": 19710, "\u0120accommodation": 19711, "\u0120Hughes": 19712, "\u0120aesthetic": 19713, "artisan": 19714, "\u0120Twilight": 19715, "\u0120prince": 19716, "\u0120Agriculture": 19717, "\u0120Disco": 19718, "\u0120precedent": 19719, "\u0120typing": 19720, "authorized": 19721, "Option": 19722, "\u0120Aub": 19723, "lishes": 19724, "acht": 19725, "mag": 19726, "Peter": 19727, "\u0120UFO": 19728, "monton": 19729, "\u0120Lith": 19730, "\u0120arom": 19731, "\u0120securing": 19732, "\u0120confined": 19733, "private": 19734, "\u0120swords": 19735, "\u0120markers": 19736, "\u0120metabolic": 19737, "select": 19738, "\u0120Curse": 19739, "\u0120Ot": 19740, "gressive": 19741, "\u0120incumb": 19742, "\u0120Saga": 19743, "\u0120priced": 19744, "\u0120clearance": 19745, "Content": 19746, "\u0120drilling": 19747, "\u0120notices": 19748, "\u0120bourgeois": 19749, "\u0120vest": 19750, "\u0120cookie": 19751, "\u0120Guardians": 19752, "rys": 19753, "inyl": 19754, "\u0120124": 19755, "\u0120plausible": 19756, "ongh": 19757, "\u0120Odin": 19758, "\u0120conception": 19759, "\u0120Yuk": 19760, "\u0120Baghdad": 19761, "\u0120Flag": 19762, "Austral": 19763, "\u0120IBM": 19764, "\u0120internationally": 19765, "\u0120WikiLeaks": 19766, "IED": 19767, "\u0120cyn": 19768, "\u0120chooses": 19769, "\u0120Pill": 19770, "\u0120combining": 19771, "\u0120radi": 19772, "\u0120Mohammed": 19773, "defense": 19774, "atching": 19775, "Subject": 19776, "iciency": 19777, "Frame": 19778, "\u0120{\"": 19779, "\u0120chess": 19780, "\u0120timer": 19781, "190": 19782, "\u0120tin": 19783, "\u0120ordinance": 19784, "emetery": 19785, "\u0120accusing": 19786, "\u0120noticeable": 19787, "\u0120centres": 19788, "\u0120lid": 19789, "\u0120Mills": 19790, "imgur": 19791, "\u0120zoom": 19792, "ergic": 19793, "\u0120compression": 19794, "prim": 19795, "find": 19796, "\u0120surg": 19797, "\u0120pand": 19798, "\u0120Kee": 19799, "\u0120Chad": 19800, "cellence": 19801, "oyle": 19802, "\u0120socialism": 19803, "\u0120Travis": 19804, "\u0120MHz": 19805, "\u0120guild": 19806, "ALLY": 19807, "\u0120Subscribe": 19808, "\u0120Related": 19809, "\u0120occurrence": 19810, "itching": 19811, "\u0120fictional": 19812, "\u0120crush": 19813, "\u0120EA": 19814, "cod": 19815, "mix": 19816, "\u0120Triple": 19817, "\u0120retrieve": 19818, "\u0120stimulus": 19819, "\u0120psychiat": 19820, "\u0120Door": 19821, "\u0120homosexuality": 19822, "\u0120elementary": 19823, "\u0120cellular": 19824, "idian": 19825, "\u0120Laun": 19826, "\u0120intriguing": 19827, "\u0120foam": 19828, "\u0120Bass": 19829, "idi": 19830, "itsu": 19831, "\u0120assure": 19832, "\u0120congrat": 19833, "\u0120businessman": 19834, "\u0120Boost": 19835, "close": 19836, "\u0120lied": 19837, "\u0120sciences": 19838, "\u0120Omega": 19839, "\u0120Graphics": 19840, "\u0120<=": 19841, "spoken": 19842, "\u0120connectivity": 19843, "Saturday": 19844, "\u0120Avengers": 19845, "\u0120toggle": 19846, "\u0120ankle": 19847, "\u0120nationalist": 19848, "model": 19849, "\u0120Pool": 19850, "ophobia": 19851, "Var": 19852, "\u0120Mons": 19853, "atories": 19854, "\u0120aggressively": 19855, "Clear": 19856, "Forge": 19857, "acters": 19858, "\u0120hedge": 19859, "\u0120pipes": 19860, "\u0120blunt": 19861, "\u0120sq": 19862, "\u0120remotely": 19863, "Wed": 19864, "asers": 19865, "\u0120refriger": 19866, "\u0120tiles": 19867, "\u0120rescued": 19868, "\u0120comprised": 19869, "insky": 19870, "\u0120manif": 19871, "avanaugh": 19872, "\u0120prolifer": 19873, "\u0120aligned": 19874, "xml": 19875, "\u0120triv": 19876, "\u0120coordination": 19877, "\u0120PER": 19878, "\u0120Quote": 19879, "134": 19880, "bf": 19881, "\u0120Saw": 19882, "\u0120termination": 19883, "\u0120190": 19884, "\u0120additions": 19885, "\u0120trio": 19886, "\u0120projections": 19887, "\u0120positively": 19888, "\u0120inclusive": 19889, "\u0120membr": 19890, "1990": 19891, "older": 19892, "\u0120practiced": 19893, "inkle": 19894, "Arch": 19895, "\u0120starters": 19896, "arius": 19897, "\u0120intermediate": 19898, "\u0120Benef": 19899, "\u0120Killer": 19900, "\u0120interventions": 19901, "\u0120Kil": 19902, "\u0120Flying": 19903, "Inv": 19904, "\u0120premature": 19905, "\u0120psychiatric": 19906, "\u0120indie": 19907, "\u0120collar": 19908, "\u0120Rainbow": 19909, "afi": 19910, "\u0120disruption": 19911, "\u0120FOX": 19912, "casting": 19913, "\u0120misdem": 19914, "cro": 19915, "\u0120wipe": 19916, "ardon": 19917, "\u0120bast": 19918, "\u0120Tommy": 19919, "\u0120Representative": 19920, "\u0120belly": 19921, "\u0120PO": 19922, "\u0120Breitbart": 19923, "132": 19924, "\u0120messaging": 19925, "Should": 19926, "References": 19927, "\u0120GRE": 19928, "istical": 19929, "LP": 19930, "\u0120Cav": 19931, "\u0120Crazy": 19932, "\u0120intuitive": 19933, "keeping": 19934, "\u0120Moss": 19935, "\u0120discontin": 19936, "\u0120Module": 19937, "\u0120unrelated": 19938, "\u0120Practice": 19939, "\u0120Transport": 19940, "\u0120statistically": 19941, "orns": 19942, "\u0120sized": 19943, "pu": 19944, "\u0120caf": 19945, "\u0120Worlds": 19946, "\u0120Rodgers": 19947, "\u0120Lun": 19948, "\u0120Comic": 19949, "living": 19950, "\u0120cared": 19951, "\u0120climbed": 19952, "){": 19953, "\u0120consisted": 19954, "\u0120medieval": 19955, "folk": 19956, "\u0120hacked": 19957, "\u0120dire": 19958, "\u0120Hermione": 19959, "\u0120tended": 19960, "ceans": 19961, "Daniel": 19962, "went": 19963, "\u0120legislators": 19964, "\u0120redes": 19965, "games": 19966, "\u0120gn": 19967, "amiliar": 19968, "\u0120++": 19969, "ggy": 19970, "threat": 19971, "\u0120magnet": 19972, "\u0120perceive": 19973, "\u0120zip": 19974, "\u0120indictment": 19975, "\u0120critique": 19976, "gard": 19977, "\u0120Safe": 19978, "\u0120Cream": 19979, "\u0120advent": 19980, "oba": 19981, "\u0120vowed": 19982, "ousands": 19983, "\u0120ski": 19984, "\u0120abortions": 19985, "uart": 19986, "\u0120stunned": 19987, "\u0120advancing": 19988, "\u0120lacked": 19989, "\u0120\\\"": 19990, "\u0120schizophren": 19991, "\u0120elegant": 19992, "\u0120conferences": 19993, "\u0120canceled": 19994, "\u0120Hudson": 19995, "\u0120Hopefully": 19996, "\u0120trump": 19997, "\u0120frequencies": 19998, "\u0120meteor": 19999, "\u0120Junior": 20000, "\u0120Fleet": 20001, "\u0120Malcolm": 20002, "\u0120Tools": 20003, "\u0120........": 20004, "\u0120hobby": 20005, "\u0120Europeans": 20006, "\u01201500": 20007, "\u0120Into": 20008, "\u0120sway": 20009, "\u0120Appro": 20010, "\u0120Compl": 20011, "Community": 20012, "\u0120tide": 20013, "\u0120Summit": 20014, "\u00e4\u00bb": 20015, "\u0120intervals": 20016, "\u0120Ether": 20017, "\u0120habitat": 20018, "\u0120Stevens": 20019, "lishing": 20020, "\u0120Domain": 20021, "\u0120triggers": 20022, "\u0120chasing": 20023, "\u0120charm": 20024, "\u0120Flower": 20025, "itored": 20026, "\u0120blessing": 20027, "\u0120textures": 20028, "Five": 20029, "\u0120liquor": 20030, "RP": 20031, "FIN": 20032, "\u01201962": 20033, "CAR": 20034, "Unknown": 20035, "\u0120resil": 20036, "\u0120Lily": 20037, "\u0120abundance": 20038, "\u0120predictable": 20039, "rar": 20040, "\u0120bullshit": 20041, "leen": 20042, "chet": 20043, "Mor": 20044, "Much": 20045, "\u00e4\u00b9": 20046, "\u0120emphasized": 20047, "\u0120crust": 20048, "\u0120primitive": 20049, "\u0120enjoyable": 20050, "\u0120Pictures": 20051, "\u0120teammate": 20052, "pler": 20053, "\u0120Tol": 20054, "\u0120Kane": 20055, "\u0120summoned": 20056, "thy": 20057, "rama": 20058, "\u0120Honda": 20059, "\u0120realizing": 20060, "\u0120quicker": 20061, "\u0120concentrate": 20062, "clear": 20063, "\u0120210": 20064, "\u0120Erdogan": 20065, "aris": 20066, "\u0120responds": 20067, "\u0120BI": 20068, "\u0120eligibility": 20069, "\u0120pushes": 20070, "\u0120Idaho": 20071, "\u0120aggrav": 20072, "\u0120ruins": 20073, "urations": 20074, "\u0120bans": 20075, "\u0120anat": 20076, "share": 20077, "\u0120grind": 20078, "hin": 20079, "umen": 20080, "\u0120utilities": 20081, "\u0120Yankees": 20082, "\u0120databases": 20083, "\u0120DD": 20084, "\u0120displaced": 20085, "\u0120dependencies": 20086, "\u0120stimulation": 20087, "hun": 20088, "houses": 20089, "\u0120Pretty": 20090, "\u0120Ravens": 20091, "\u0120TODAY": 20092, "\u0120associates": 20093, "\u0120therape": 20094, "cled": 20095, "\u0120deer": 20096, "\u0120repairs": 20097, "rentice": 20098, "\u0120receptors": 20099, "\u0120remed": 20100, "\u0120Ce": 20101, "\u0120marriages": 20102, "\u0120ballots": 20103, "\u0120Soldier": 20104, "\u0120hilarious": 20105, "opl": 20106, "138": 20107, "\u0120inherently": 20108, "\u0120ignorant": 20109, "\u0120bounce": 20110, "\u0120Easter": 20111, "RELATED": 20112, "\u0120Currency": 20113, "EV": 20114, "\u00e3\u0125\u0140": 20115, "\u0120Lead": 20116, "\u0120deceased": 20117, "Brien": 20118, "\u0120Musk": 20119, "JS": 20120, "\u0120merge": 20121, "hearted": 20122, "creat": 20123, "mitt": 20124, "mund": 20125, "\u0120\u00e2\u0122\u012d": 20126, "\u0120Bag": 20127, "\u0120projection": 20128, "\u0120java": 20129, "\u0120Standards": 20130, "\u0120Leonard": 20131, "\u0120coconut": 20132, "\u0120Population": 20133, "\u0120traject": 20134, "\u0120imply": 20135, "\u0120curiosity": 20136, "\u0120DB": 20137, "\u0120Fresh": 20138, "\u0120Por": 20139, "\u0120heavier": 20140, "neys": 20141, "gomery": 20142, "\u0120deserved": 20143, "\u0120phrases": 20144, "\u0120GC": 20145, "\u0120yeast": 20146, "desc": 20147, "Death": 20148, "\u0120reboot": 20149, "\u0120metadata": 20150, "ICAL": 20151, "\u0120repay": 20152, "\u0120Independence": 20153, "\u0120suburban": 20154, "icals": 20155, "\u0120atop": 20156, "\u0120allocation": 20157, "generation": 20158, "\u0120Gram": 20159, "\u0120moisture": 20160, "\u0120pine": 20161, "\u0120Liberals": 20162, "\u0120aides": 20163, "\u0120underest": 20164, "\u0120Berry": 20165, "\u0120ceremon": 20166, "370": 20167, "astrous": 20168, "\u0120Pirates": 20169, "\u0120tense": 20170, "\u0120Industries": 20171, "\u0120Appeals": 20172, "\u0120Near": 20173, "\u0120\u00e8\u00a3\u0131\u00e7": 20174, "\u0120lovers": 20175, "\u0120CAP": 20176, "\u0120Craw": 20177, "\u0120giants": 20178, "\u0120efficacy": 20179, "Element": 20180, "\u0120Behavior": 20181, "\u0120Toyota": 20182, "\u0120intest": 20183, "Priv": 20184, "AI": 20185, "\u0120maneuver": 20186, "\u0120perfection": 20187, "\u0120bang": 20188, "paper": 20189, "rill": 20190, "George": 20191, "border": 20192, "inters": 20193, "\u0120Seth": 20194, "\u0120clues": 20195, "\u0120Levi": 20196, "\u0120Revenue": 20197, "147": 20198, "\u0120vapor": 20199, "\u0120fortunate": 20200, "\u0120threatens": 20201, "\u0120vet": 20202, "\u0120dependency": 20203, "ersed": 20204, "article": 20205, "\u0120Blizzard": 20206, "\u0120chlor": 20207, "\u0120minus": 20208, "\u0120Bills": 20209, "\u0120cryptocurrency": 20210, "\u0120metabolism": 20211, "tering": 20212, "\u0120pestic": 20213, "steps": 20214, "\u0120Treasure": 20215, "racted": 20216, "\u0120Constant": 20217, "\u0120temp": 20218, "139": 20219, "\u0120Detective": 20220, "urally": 20221, "\u0120recovering": 20222, "\u0120cortex": 20223, "\u0120144": 20224, "closed": 20225, "\u0120prejudice": 20226, "aunted": 20227, "\u0120storms": 20228, "\u0120NOW": 20229, "\u0120machinery": 20230, "Address": 20231, "\u0120compelled": 20232, "270": 20233, "\u0120despair": 20234, "bane": 20235, "\u0120vegetable": 20236, "\u0120beds": 20237, "Learn": 20238, "\u0120colorful": 20239, "\u0120spike": 20240, "\u0120margins": 20241, "\u0120sympathy": 20242, "\u0120workshop": 20243, "\u0120CBC": 20244, "Sat": 20245, "\u0120burns": 20246, "\u0120Gender": 20247, "\u0120129": 20248, "\u0120Cable": 20249, "\u0120debts": 20250, "\u0120Theresa": 20251, "\u0120reflecting": 20252, "\u0120airst": 20253, "\u0120rim": 20254, "ramid": 20255, "\u0120weaknesses": 20256, "Writ": 20257, "oggle": 20258, "ti": 20259, "\u0120Charge": 20260, "\u0120weighed": 20261, "\u0120(.": 20262, "\u0120laughter": 20263, "\u0120router": 20264, "\u0120Democracy": 20265, "Dear": 20266, "\u0120hasht": 20267, "\u0120dy": 20268, "\u0120hints": 20269, "running": 20270, "\u0120finishes": 20271, "arus": 20272, "Mass": 20273, "result": 20274, "ascus": 20275, "\u0120vintage": 20276, "\u0120conqu": 20277, "\u0120wildly": 20278, "acist": 20279, "\u0120lingu": 20280, "\u0120protagonist": 20281, "strom": 20282, "teenth": 20283, "\u0120Solo": 20284, "mac": 20285, "filled": 20286, "\u0120renown": 20287, "itives": 20288, "\u0120motive": 20289, "\u0120Antar": 20290, "\u0120Mann": 20291, "\u0120Adjust": 20292, "\u0120rockets": 20293, "\u0120troubling": 20294, "ei": 20295, "\u0120organisms": 20296, "assis": 20297, "Christian": 20298, "\u0120145": 20299, "\u0120Hass": 20300, "\u0120swall": 20301, "\u0120wax": 20302, "\u0120Survival": 20303, "VS": 20304, "\u0120Murd": 20305, "vd": 20306, "standard": 20307, "\u0120dragons": 20308, "\u0120acceleration": 20309, "rational": 20310, "final": 20311, "\u0120paired": 20312, "\u0120Ethereum": 20313, "\u0120interfaces": 20314, "\u0120resent": 20315, "\u0120artifacts": 20316, "\u00c5\u00ab": 20317, "arel": 20318, "\u0120competitor": 20319, "\u0120Nicholas": 20320, "\u0120Surface": 20321, "cpp": 20322, "\u0120Tot": 20323, "\u0120economically": 20324, "\u0120organised": 20325, "\u0120enforced": 20326, "inho": 20327, "\u0120varieties": 20328, "\u0120abdom": 20329, "\u0120Bailey": 20330, "idav": 20331, "\u0120Salv": 20332, "paid": 20333, "\u0120altitude": 20334, "essert": 20335, "\u0120Gutenberg": 20336, "area": 20337, "opoulos": 20338, "\u0120professors": 20339, "iggs": 20340, "\u0120Fate": 20341, "hey": 20342, "\u01203000": 20343, "Dist": 20344, "\u0120twins": 20345, "cill": 20346, "\u0120Maps": 20347, "\u0120traps": 20348, "\u0120weed": 20349, "\u0120Kiss": 20350, "\u0120yoga": 20351, "\u0120recipients": 20352, "\u0120Westminster": 20353, "\u0120pools": 20354, "\u0120Walmart": 20355, "188": 20356, "\u0120Schools": 20357, "attack": 20358, "\u0120ARM": 20359, "paragraph": 20360, "Warning": 20361, "jl": 20362, "\u0120selfish": 20363, "anchez": 20364, "\u0120Heights": 20365, "Fre": 20366, "\u0120Soph": 20367, "\u0120--------------------------------": 20368, "tml": 20369, "333": 20370, "\u0120raids": 20371, "\u0120satellites": 20372, "KEY": 20373, "\u0120lasts": 20374, "\u00d1\u0124": 20375, "Ins": 20376, "\u0120Dame": 20377, "\u0120unpredict": 20378, "///": 20379, "ghai": 20380, "\u0120artillery": 20381, "\u0120cruise": 20382, "\u0120gel": 20383, "\u0120Cabinet": 20384, "\u0120blows": 20385, "\u0120Esp": 20386, "\u0120proximity": 20387, "othe": 20388, "\u0120Skills": 20389, "\u0120Upper": 20390, "obo": 20391, "\u0120NDP": 20392, "\u0120enjoys": 20393, "\u0120repeating": 20394, "\u0120Construction": 20395, "\u0120Questions": 20396, "Hillary": 20397, "\u0120uint": 20398, "\u0120processors": 20399, "\u0120Gibson": 20400, "\u0120Multiple": 20401, "qa": 20402, "\u0120Bom": 20403, "\u0120Miles": 20404, "ventional": 20405, "\u0120hurts": 20406, "skin": 20407, "\u0120AIDS": 20408, "\u0120advisers": 20409, "\u0120Root": 20410, "\u0120methodology": 20411, "\u0120Dale": 20412, "\u0120deton": 20413, "\u0120Knowledge": 20414, "sequently": 20415, "\u0120121": 20416, "\u0120connects": 20417, "Cy": 20418, "\u0120Danger": 20419, "\u0120contributors": 20420, "\u0120Bent": 20421, "\u0120brass": 20422, "\u0120Guns": 20423, "into": 20424, "\u0120Fortune": 20425, "\u0120broker": 20426, "balance": 20427, "\u0120lengths": 20428, "\u0120vic": 20429, "\u0120averaging": 20430, "\u0120appropriately": 20431, "\u0120Camera": 20432, "\u0120sandwich": 20433, "\u0120CDC": 20434, "\u0120coordinate": 20435, "\u0120navig": 20436, "\u0120goodness": 20437, "laim": 20438, "\u0120brake": 20439, "\u0120extremist": 20440, "\u0120Wake": 20441, "\u0120Mend": 20442, "\u0120Tiny": 20443, "\u0120COL": 20444, "\u0120RF": 20445, "\u0120Dual": 20446, "\u0120Wine": 20447, "Case": 20448, "\u0120refined": 20449, "\u0120lamp": 20450, "Lead": 20451, "\u0120bapt": 20452, "\u0120Carb": 20453, "\u0120Sadd": 20454, "\u0120Minneapolis": 20455, "PDF": 20456, "Early": 20457, "\u0120Hidden": 20458, "Its": 20459, "\u0120TIME": 20460, "\u0120pap": 20461, "\u0120commissioned": 20462, "\u0120Few": 20463, "\u0120Colts": 20464, "\u0120Bren": 20465, "\u0120bothered": 20466, "\u0120likewise": 20467, "Exper": 20468, "\u0120Schw": 20469, "cry": 20470, "nn": 20471, "\u0120Mitch": 20472, "imon": 20473, "MG": 20474, "bm": 20475, "UMP": 20476, "rays": 20477, "\u0120registry": 20478, "\u0120270": 20479, "achine": 20480, "rella": 20481, "anting": 20482, "00000": 20483, "\u0120ruined": 20484, "spot": 20485, "\u0120ta": 20486, "\u0120maximize": 20487, "\u0120inconven": 20488, "Dead": 20489, "Human": 20490, "Enabled": 20491, "\u0120Marie": 20492, "\u0120chill": 20493, "\u0120Paradise": 20494, "\u0120starring": 20495, "\u0120Latino": 20496, "\u0120Protocol": 20497, "\u0120EVER": 20498, "\u0120suppliers": 20499, "message": 20500, "\u0120Brock": 20501, "\u0120serum": 20502, "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 20503, "\u0120encomp": 20504, "\u0120ambition": 20505, "uese": 20506, "\u0120arrows": 20507, "Andrew": 20508, "\u0120antenna": 20509, "\u01201961": 20510, "\u0120Bark": 20511, "\u0120bool": 20512, "\u00e3\u0124\u00aa": 20513, "\u0120Storage": 20514, "\u0120railway": 20515, "\u0120tougher": 20516, "\u0120Cad": 20517, "\u0120washing": 20518, "Py": 20519, "']": 20520, "embed": 20521, "\u0120Memphis": 20522, "ackle": 20523, "\u0120famously": 20524, "\u0120Fortunately": 20525, "ovies": 20526, "\u0120mindset": 20527, "\u0120sneak": 20528, "\u0120Dh": 20529, "RAW": 20530, "\u0120Simpson": 20531, "\u0120livest": 20532, "\u0120landmark": 20533, "\u0120cement": 20534, "Low": 20535, "\u0120thrilled": 20536, "\u0120Course": 20537, "inel": 20538, "\u0120chuck": 20539, "idate": 20540, "global": 20541, "\u0120whit": 20542, "\u0120\u00ef\u00bf\u00bd": 20543, "adays": 20544, "ski": 20545, "\u0120SV": 20546, "\u0120viruses": 20547, "306": 20548, "\u0120Respons": 20549, "\u0120theaters": 20550, "\u0120Branch": 20551, "\u0120Geneva": 20552, "\u0120MK": 20553, "\u0120unbeliev": 20554, "\u0120communist": 20555, "Original": 20556, "\u0120Received": 20557, "\u0120Transfer": 20558, "\u0120Arg": 20559, "Input": 20560, "\u0120Strategy": 20561, "\u0120palace": 20562, "thening": 20563, "Dri": 20564, "\u0120sentencing": 20565, "umbnail": 20566, "\u0120pins": 20567, "recy": 20568, "\u0120siblings": 20569, "Getting": 20570, "\u0120BU": 20571, "\u0120Northwest": 20572, "\u0120prolonged": 20573, "\u0120Sakura": 20574, "Comb": 20575, "\u0120Bour": 20576, "\u0120inadequate": 20577, "\u0120Kash": 20578, "\u0120username": 20579, "\u0120Improve": 20580, "\u0120battling": 20581, "\u0120MAC": 20582, "\u0120curriculum": 20583, "\u0120soda": 20584, "\u0120Cannon": 20585, "\u0120sensible": 20586, "spons": 20587, "December": 20588, "\u0120wicked": 20589, "\u0120Pengu": 20590, "\u0120dictators": 20591, "\u0120Hearts": 20592, "ogyn": 20593, "\u0120similarities": 20594, "\u0120Stats": 20595, "\u0120hollow": 20596, "itations": 20597, "\":[": 20598, "\u0120hover": 20599, "\u0120Listen": 20600, "sch": 20601, "Sund": 20602, "\u0120cad": 20603, "\u0120Parks": 20604, "\u0120lur": 20605, "\u0120hype": 20606, "\u0120Lem": 20607, "NAME": 20608, "isure": 20609, "Friday": 20610, "\u0120shoots": 20611, "\u0120closes": 20612, "\u0120db": 20613, "\u0120Ridge": 20614, "\u0120Different": 20615, "\u0120replies": 20616, "\u0120Broadway": 20617, "opers": 20618, "\u0120intoler": 20619, "\u0120Zeus": 20620, "akespe": 20621, "\u0120proprietary": 20622, "\u0120requesting": 20623, "\u0120controllers": 20624, "\u0120MIN": 20625, "imedia": 20626, "becca": 20627, "\u0120expans": 20628, "\u0120oils": 20629, "Bot": 20630, "\u0120Chand": 20631, "\u0120printer": 20632, "\u0120topped": 20633, "\u0120POL": 20634, "\u0120Earlier": 20635, "Social": 20636, "avin": 20637, "\u0120decreases": 20638, "\u0120Seb": 20639, "\u0120specifications": 20640, "\u0120Blast": 20641, "\u0120Kurt": 20642, "\u0120freel": 20643, "Brown": 20644, "\u0120dilig": 20645, "roe": 20646, "\u0120Problem": 20647, "\u0120Quad": 20648, "\u0120decentral": 20649, "\u0120Vector": 20650, "anut": 20651, "\u0120plugins": 20652, "\u0120Gregory": 20653, "\u0120fucked": 20654, "elines": 20655, "\u0120Ambassador": 20656, "take": 20657, "\u0120cleans": 20658, "ongyang": 20659, "Anonymous": 20660, "stro": 20661, "\"}": 20662, "aline": 20663, "\u0120Odd": 20664, "\u0120Eug": 20665, "216": 20666, "\u0120boil": 20667, "\u0120Powers": 20668, "\u0120nurses": 20669, "Obviously": 20670, "\u0120Technical": 20671, "\u0120exceeded": 20672, "ORS": 20673, "\u0120extremists": 20674, "\u0120traces": 20675, "expl": 20676, "\u0120comr": 20677, "\u0120Sach": 20678, ")/": 20679, "\u0120masks": 20680, "\u0120sci": 20681, "Bon": 20682, "\u0120regression": 20683, "wegian": 20684, "\u0120advisor": 20685, "itures": 20686, "\u0120Vo": 20687, "example": 20688, "\u0120Instruct": 20689, "\u0120siege": 20690, "\u0120reductions": 20691, "ptr": 20692, "\u0120statutory": 20693, "\u0120removes": 20694, "\u0120puck": 20695, "redits": 20696, "\u0120bee": 20697, "\u0120salad": 20698, "\u0120promotions": 20699, "\u0120Joshua": 20700, "withstanding": 20701, "ETH": 20702, "\u0120Cha": 20703, "imus": 20704, "\u0120expenditure": 20705, "aunting": 20706, "\u0120delighted": 20707, "\u0120155": 20708, "beh": 20709, "\u0120carpet": 20710, "\u0120Spart": 20711, "\u0120jungle": 20712, "lists": 20713, "\u0120bullying": 20714, "\u0120Nobel": 20715, "\u0120Glen": 20716, "\u0120referenced": 20717, "\u0120introduces": 20718, "sein": 20719, "\u0120chopped": 20720, "glass": 20721, "\u0120Wrest": 20722, "\u0120neutrality": 20723, "\u0120\u00e2\u013b": 20724, "\u0120investigator": 20725, "\u0120shelves": 20726, "\u0120unconstitutional": 20727, "\u0120reproduction": 20728, "\u0120merchant": 20729, "mia": 20730, "\u0120metrics": 20731, "\u0120explosives": 20732, "\u0120Sonia": 20733, "\u0120bodily": 20734, "\u0120thickness": 20735, "\u0120predominantly": 20736, "\u0120Ability": 20737, "\u0120monitored": 20738, "ICH": 20739, "\u0120].": 20740, "\u0120Martinez": 20741, "\u0120visibility": 20742, "\u0120queries": 20743, "\u0120genocide": 20744, "\u0120Warfare": 20745, "Query": 20746, "\u0120studios": 20747, "\u0120embry": 20748, "\u0120corridor": 20749, "\u0120cleaned": 20750, "complete": 20751, "\u0120MH": 20752, "\u0120enrollment": 20753, "INGS": 20754, "\u0120impacted": 20755, "\u0120disastrous": 20756, "\u0120Yun": 20757, "\u0120Claire": 20758, "\u0120Basically": 20759, "yt": 20760, "usterity": 20761, "\u0120indirectly": 20762, "wik": 20763, "\u0120dod": 20764, "\u0120Carr": 20765, "\u0120amp": 20766, "\u0120prohibit": 20767, "\u0120Initial": 20768, "\u0120Rd": 20769, "iji": 20770, "\u0120educate": 20771, "corn": 20772, "iott": 20773, "\u0120Beauty": 20774, "\u0120detective": 20775, "\u0120Conn": 20776, "since": 20777, "\u0120stagger": 20778, "\u0120obese": 20779, "\u0120bree": 20780, "ologic": 20781, "isse": 20782, "walker": 20783, "\u0120blades": 20784, "\u0120lawful": 20785, "func": 20786, "\u0120Behind": 20787, "\u0120appetite": 20788, "\u0120(*": 20789, "\u0120tennis": 20790, "\u0120offspring": 20791, "\u0120jets": 20792, "\u0120structured": 20793, "\u0120aforementioned": 20794, "Nov": 20795, "\u0120scaling": 20796, "fill": 20797, "\u0120stew": 20798, "\u0120curb": 20799, "\u0120Stephan": 20800, "edIn": 20801, "SF": 20802, "obic": 20803, "\u00e9\u0143\u0136": 20804, "oug": 20805, "\u0120MM": 20806, "\u0120genetically": 20807, "opez": 20808, "136": 20809, "\u0120umb": 20810, "ancers": 20811, "\u0120cohort": 20812, "\u0120merchandise": 20813, "\u0120imposing": 20814, "\u0120Legislature": 20815, "\u0120Archive": 20816, "ivia": 20817, "\u0120Naval": 20818, "\u0120offences": 20819, "\u0120miracle": 20820, "\u0120snapped": 20821, "\u0120foes": 20822, "\u0120extensively": 20823, "\u0120Raf": 20824, "\u0120cater": 20825, "edience": 20826, "Kit": 20827, "\u0120Bin": 20828, "\u0120recommends": 20829, "\u0120Cities": 20830, "\u0120rigid": 20831, "\u0120READ": 20832, "\u0120Noble": 20833, "\u0120Tian": 20834, "\u0120certificates": 20835, "antis": 20836, "oiler": 20837, "\u0120Buddhist": 20838, "did": 20839, "\u0120surveyed": 20840, "\u0120downward": 20841, "\u0120prints": 20842, "\u0120Motion": 20843, "ronics": 20844, "\u0120Sans": 20845, "ossibly": 20846, "uctions": 20847, "\u0120colonies": 20848, "\u0120Danish": 20849, "unit": 20850, "\u0120spoil": 20851, "\u0120advisory": 20852, "berries": 20853, "Plan": 20854, "\u0120specification": 20855, "ophers": 20856, "\u0120Resource": 20857, "\u0120shirts": 20858, "prisingly": 20859, "communications": 20860, "\u0120trivial": 20861, "\u0120mentioning": 20862, "isexual": 20863, "\u0120supplements": 20864, "\u0120supervision": 20865, "BP": 20866, "vor": 20867, "\u0120wit": 20868, "\u0120cooldown": 20869, "\u0120plaintiff": 20870, "\u0120Reviews": 20871, "\u0120Sri": 20872, "\u0120Mint": 20873, "\u0120Sugar": 20874, "\u0120afterward": 20875, "\u0120Priest": 20876, "\u0120Investment": 20877, "ogene": 20878, "\u0120Taking": 20879, "\u0120stretching": 20880, "\u0120inflammation": 20881, "\u0120Tehran": 20882, "\u0120lining": 20883, "\u0120freezing": 20884, "\u0120Entity": 20885, "\u0120inspiring": 20886, "special": 20887, "price": 20888, "\u0120sue": 20889, "\u0120Porter": 20890, "ounge": 20891, "ETA": 20892, "\u0120Derek": 20893, "\u0120Luis": 20894, "uo": 20895, "ymph": 20896, "\u0120exterior": 20897, "ihil": 20898, "\u0120Ashley": 20899, "inator": 20900, "\u0120nutrients": 20901, "\u0120Thrones": 20902, "\u0120finances": 20903, "\u0120Inspect": 20904, "\u0120specially": 20905, "\u0120Required": 20906, "\u0120PTS": 20907, "\u0120Violence": 20908, "ointed": 20909, "shots": 20910, "\u0120excerpt": 20911, "coon": 20912, "INS": 20913, "\u0120Gri": 20914, "\u0120recognised": 20915, "Week": 20916, "Young": 20917, "\u0120vom": 20918, "isle": 20919, "\u0120Curry": 20920, "\u0120Buddh": 20921, "\u0120notebook": 20922, "\u0120durable": 20923, "/?": 20924, "\u0120Gad": 20925, "\u0120Pupp": 20926, "\u0120forgive": 20927, "park": 20928, "\u0120personalities": 20929, "analysis": 20930, "clamation": 20931, "\u0120elevator": 20932, "\u0120warehouse": 20933, "\u0120Role": 20934, "unn": 20935, "\u0120illustration": 20936, "\u0120Scan": 20937, "\u0120atmospheric": 20938, "Import": 20939, "ANC": 20940, "ricted": 20941, "fu": 20942, "010": 20943, "\u0120arche": 20944, "\u0120rewarded": 20945, "akespeare": 20946, "\u0120internally": 20947, "\u0120RBI": 20948, "alker": 20949, "\u0120elephant": 20950, "owitz": 20951, "\u0120Pizza": 20952, "\u0120bipartisan": 20953, "\u00c3\u00a9s": 20954, "\u0120slowed": 20955, "\u0120Stark": 20956, "\u0120override": 20957, "OUS": 20958, "\u0120320": 20959, "undreds": 20960, "\u0120Deck": 20961, "\u0120Census": 20962, "bee": 20963, "146": 20964, "otor": 20965, "\u0120ip": 20966, "\u0120ub": 20967, "ocations": 20968, "\u0120Button": 20969, "rice": 20970, "\u0120cripp": 20971, "fff": 20972, "\u0120originated": 20973, "\u0120overwhelmed": 20974, "appa": 20975, "\u0120foremost": 20976, "\u00e2\u0122\u0133": 20977, "\u0120LEG": 20978, "release": 20979, "eatured": 20980, "atches": 20981, "\u0120reps": 20982, "\u0120lending": 20983, "\u0120Reference": 20984, "\u0120Client": 20985, "165": 20986, "venth": 20987, "Complete": 20988, "\u0120Patrol": 20989, "\u0120sworn": 20990, "cam": 20991, "\u0120shuttle": 20992, "\u0120Ralph": 20993, "\u0120hometown": 20994, "-,": 20995, "onal": 20996, "\u0120BP": 20997, "\u00e5\u0131": 20998, "\u0120persuade": 20999, "\u0120Alexand": 21000, "\u0120combines": 21001, "\u0120vivid": 21002, "\u0120Lag": 21003, "\u0120encoding": 21004, "\u0120salvation": 21005, "wen": 21006, "\u0120Recovery": 21007, "iya": 21008, "University": 21009, "\u0120Biden": 21010, "\u0120budgets": 21011, "\u0120Texans": 21012, "fits": 21013, "\u0120honored": 21014, "\u0120python": 21015, "TD": 21016, "###": 21017, "clone": 21018, "\u0120blink": 21019, "\u0120Liquid": 21020, "\u0120unemployed": 21021, "\u0120clashes": 21022, "\u0120Counsel": 21023, "\u0120directing": 21024, "\u0120punct": 21025, "\u0120Falcons": 21026, "\u0120shark": 21027, "\u0120Damascus": 21028, "\u0120jeans": 21029, "\u0120embark": 21030, "\u0120seize": 21031, "\u0120upwards": 21032, "280": 21033, "\u0120Ez": 21034, "\u0120Anything": 21035, "\u0120exotic": 21036, "lower": 21037, "\u0120Creator": 21038, "\u0120Um": 21039, "\u0120suburbs": 21040, "berger": 21041, "\u0120Wend": 21042, "\u0120mint": 21043, "\u0120XX": 21044, "\u0120Dro": 21045, "\u0120suffers": 21046, "\u0120herb": 21047, "tree": 21048, "\u0120fragile": 21049, "\u0120flooded": 21050, "\u0120Alcohol": 21051, "olean": 21052, "nyder": 21053, "\u0120KO": 21054, "Fram": 21055, "\u0120136": 21056, "\u0120owed": 21057, "\u0120Melee": 21058, "\u0120Hash": 21059, "\u0120whisk": 21060, "\u0120sudo": 21061, "rr": 21062, "Quick": 21063, "appro": 21064, "\u0120ii": 21065, "\u0120Examples": 21066, "hee": 21067, "\u0120promotes": 21068, "perature": 21069, "kar": 21070, "\u0120Honor": 21071, "\u0120sodium": 21072, "\u0120Lif": 21073, "rosso": 21074, "intendent": 21075, "\u0120correspondent": 21076, "Found": 21077, "secret": 21078, "\u0120identifies": 21079, "agne": 21080, "\u0120lou": 21081, "\u0120PP": 21082, "\u0120coincidence": 21083, "move": 21084, "\u0120militia": 21085, "\u0120infiltr": 21086, "\u0120Primary": 21087, "\u0120pitching": 21088, "\u0120Ib": 21089, "\u0120GOOD": 21090, "\u00e3\u0124\u00b8": 21091, "\u0120Wizards": 21092, "iral": 21093, "\u0120Venus": 21094, "RR": 21095, "\u0120\u00e2\u0122\u0137": 21096, "\u0120Casey": 21097, "\u0120sadly": 21098, "\u0120admire": 21099, "\u0120embarrassed": 21100, "cb": 21101, "Mel": 21102, "\u0120tubes": 21103, "\u0120beautifully": 21104, "\u0120Queensland": 21105, "Below": 21106, "rez": 21107, "quet": 21108, "pleasant": 21109, "\u0120\u00c2\u00ab": 21110, "Camp": 21111, "\u0120decisive": 21112, "1998": 21113, "\u0120Lamb": 21114, "utton": 21115, "hn": 21116, "\u0120Jagu": 21117, "aunder": 21118, "\u0120Cord": 21119, "\u0120clerk": 21120, "\u0120caffe": 21121, "\u0120wiped": 21122, "\u0120reim": 21123, "\u0120Mountains": 21124, "\u0120imprisoned": 21125, "\u0120develops": 21126, "\u0120Pra": 21127, "\u0120modeling": 21128, "Anyone": 21129, "ancel": 21130, "\u0120Sit": 21131, "\u0120shields": 21132, "\u0120lawn": 21133, "\u0120cardiovascular": 21134, "\u0120demonstrating": 21135, "\u0120parse": 21136, "\u0120Israelis": 21137, "\u0120euros": 21138, "143": 21139, "\u0120glorious": 21140, "inski": 21141, "ecd": 21142, "\u0120conditioning": 21143, "\u0120helpless": 21144, "\u0120microsc": 21145, "\u0120Harbor": 21146, "\u0120stakes": 21147, "\u0120260": 21148, "\u0120unequ": 21149, "\u0120Floyd": 21150, "\u0120damp": 21151, "\u0120apparatus": 21152, "\u0120Laws": 21153, "\u0120counters": 21154, "\u0120induce": 21155, "atable": 21156, "\u0120Ahmed": 21157, "\u0120slam": 21158, "November": 21159, "\u0120persist": 21160, "\u0120imminent": 21161, "\u00c3\u00a1n": 21162, "\u0120shred": 21163, "\u0120phases": 21164, "\u0120Edmonton": 21165, "\u0120Armstrong": 21166, "\u0120Meet": 21167, "\u0120Kitty": 21168, "\u00d1\u0122": 21169, "circ": 21170, "\u0120Adult": 21171, "\u0120arose": 21172, "\u0120Xen": 21173, "Dan": 21174, "gow": 21175, "\u0120superf": 21176, "\u0120Admir": 21177, "\u0120endure": 21178, "\u0120keyword": 21179, "yrus": 21180, "\u0120yarn": 21181, "\u0120pathway": 21182, "\u0120Hopkins": 21183, "midt": 21184, "\u0120censorship": 21185, "dependent": 21186, "\u0120instructor": 21187, "Sources": 21188, "\u0120toe": 21189, "\u0120balloon": 21190, "Nob": 21191, "\u0120swear": 21192, "\u0120Castro": 21193, "\u0120gloss": 21194, "\u0120Kavanaugh": 21195, "\u0120remarkably": 21196, "Photos": 21197, "\u0120Nom": 21198, "\u0120Southeast": 21199, "yers": 21200, "\u0120validation": 21201, "\u0120cannon": 21202, "\u0120Victory": 21203, "\u0120Pierre": 21204, "\u0120cautious": 21205, "Audio": 21206, "\u0120fetch": 21207, "\u0120Gift": 21208, "\u0120Hyp": 21209, "\u0120remedy": 21210, "ZE": 21211, "\u0120scent": 21212, "\u0120beard": 21213, "\u0120Rut": 21214, "-\"": 21215, "\u0120patents": 21216, "Hy": 21217, "\u0120unjust": 21218, "\u0120potato": 21219, "\u0120forthcoming": 21220, "\u0120chef": 21221, "\u0120Rift": 21222, "affe": 21223, "\u0120ROM": 21224, "\u0120Launch": 21225, "\u0120pads": 21226, "\u0120Neo": 21227, "\u0120onset": 21228, "\u0120squeeze": 21229, "safe": 21230, "\u0120prefix": 21231, "\u0120TM": 21232, "\u0120Nearly": 21233, "\u0120Clinical": 21234, "\u0120Mental": 21235, "otiation": 21236, "\u0120Unic": 21237, "antry": 21238, "\u0120Cir": 21239, "\u0120epit": 21240, "\u00c3\u00a6": 21241, "\u0120extracted": 21242, "versely": 21243, "riad": 21244, "\u0120strains": 21245, "\u0120tops": 21246, "\u0120poem": 21247, "\u0120Randy": 21248, "\u0120Maple": 21249, "THER": 21250, "upiter": 21251, "\u0120SSD": 21252, "\u013c\u00e9": 21253, "\u0120uncon": 21254, "pering": 21255, "\u0120slept": 21256, "iners": 21257, "\u0120underwater": 21258, "\u0120Evidence": 21259, "gone": 21260, "205": 21261, "\u0120historians": 21262, "\u0120synthesis": 21263, "\u0120frog": 21264, "basketball": 21265, "\u0120vibrant": 21266, "\u0120subord": 21267, "\u0120365": 21268, "\u0120Dial": 21269, "\u0120cooperate": 21270, "HAHA": 21271, "\u0120greeted": 21272, "158": 21273, "\u0120jazz": 21274, "\u0120intox": 21275, "\u0120Walking": 21276, "\u0120supervisor": 21277, "\u0120Fusion": 21278, "\u0120Mercedes": 21279, "send": 21280, "Ham": 21281, "sd": 21282, "nl": 21283, "\u0120tours": 21284, "\u0120FIFA": 21285, "\u0120culp": 21286, "gd": 21287, "304": 21288, "\u0120pleas": 21289, "\u0120illustrates": 21290, "\u0120Colombia": 21291, "\u0120highlighting": 21292, "\u0120Summary": 21293, "\u0120exposing": 21294, "\u0120Dru": 21295, "\u0120irony": 21296, "ritional": 21297, "\u0120Carroll": 21298, "\u0120Ellis": 21299, "Pict": 21300, "\u0120Rapt": 21301, "\u0120adapter": 21302, "\u0120unm": 21303, "\u0120corpse": 21304, "\u0120celebrities": 21305, "Den": 21306, "atum": 21307, "\u0120Apocalypse": 21308, "\u0120Wag": 21309, "lining": 21310, "\u0120hormones": 21311, "Rub": 21312, "\u0120Xi": 21313, "\u0120Vaults": 21314, "208": 21315, "alkyrie": 21316, "inosaur": 21317, "\u0120feeds": 21318, "vity": 21319, "\u0120defeating": 21320, "Wait": 21321, "\u0120emphasize": 21322, "\u0120Steelers": 21323, "yrinth": 21324, "leys": 21325, "\u0120Whenever": 21326, "Currently": 21327, "\u0120Clock": 21328, "\u0120collectively": 21329, "anyon": 21330, "\u0120JP": 21331, "\u0120mentality": 21332, "\u0120downloads": 21333, "\u0120surroundings": 21334, "\u0120Barnes": 21335, "\u0120flagship": 21336, "\u0120indicators": 21337, "\u0120grapp": 21338, "January": 21339, "\u0120Elemental": 21340, "\u0120Athena": 21341, "ibal": 21342, "\u0120sights": 21343, "\u0120capita": 21344, "\u0120Treaty": 21345, "\u0120voiced": 21346, "\u0120Gaz": 21347, "lette": 21348, "\u0120ya": 21349, "\u0120expired": 21350, "Legend": 21351, "Hot": 21352, "nature": 21353, "\u0120unstable": 21354, "\u0120280": 21355, "\u00c3\u00ba": 21356, "Comment": 21357, "ALE": 21358, "\u0120quests": 21359, "\u0120handler": 21360, "nis": 21361, "\u0120versatile": 21362, "\u0120conceal": 21363, "engeance": 21364, "\u0120Interactive": 21365, "\u0120obsessed": 21366, "\u0120Dogs": 21367, "\u0120cracked": 21368, "Sound": 21369, "sv": 21370, "\u0120Dylan": 21371, "roads": 21372, "fx": 21373, "\u0120Catholics": 21374, "\u0120Hag": 21375, "\u0120slammed": 21376, "\u0120glowing": 21377, "sale": 21378, "\u0120tissues": 21379, "\u0120Chi": 21380, "nee": 21381, "\u0120cher": 21382, "sic": 21383, "urrection": 21384, "\u0120bacon": 21385, "ulatory": 21386, ").\"": 21387, "\u0120irregular": 21388, "FORM": 21389, "assed": 21390, "\u0120intentional": 21391, "\u0120compensate": 21392, "\u0120Speaking": 21393, "\u0120Sets": 21394, "153": 21395, "\u0120conventions": 21396, "bands": 21397, "emade": 21398, "\u0120ecc": 21399, "\u0120Winston": 21400, "\u0120Assassin": 21401, "\u0120Belgian": 21402, "\u0120dependence": 21403, "\u0120niche": 21404, "\u0120bark": 21405, "\u0120Jazz": 21406, "\u0120disadvantage": 21407, "\u0120gasoline": 21408, "\u0120165": 21409, "\u00e7\u013c\u0126": 21410, "essa": 21411, "module": 21412, "angular": 21413, "OY": 21414, "\u0120Treatment": 21415, "itas": 21416, "olation": 21417, "\u0120Arnold": 21418, "\u0120feud": 21419, "\u0120Nest": 21420, "\u0120theatre": 21421, "ewater": 21422, "\u0120minors": 21423, "olicy": 21424, "\u0120Haven": 21425, "division": 21426, "\u0120trunk": 21427, "Far": 21428, "\u0120Pull": 21429, "\u0120capturing": 21430, "\u01201800": 21431, "\u0120Teen": 21432, "\u0120exempl": 21433, "\u0120clinics": 21434, "\u0120Burg": 21435, "\u0120substit": 21436, "\u0120payload": 21437, "\u0120Lav": 21438, "\u0120Troy": 21439, "\u0120Witness": 21440, "\u0120fragments": 21441, "\u0120passwords": 21442, "\u0120gospel": 21443, "\u0120Gin": 21444, "\u0120tenants": 21445, "olith": 21446, "Six": 21447, "Previous": 21448, "\u0120Ages": 21449, "\u0120Darwin": 21450, "\u0120blat": 21451, "\u0120empathy": 21452, "smith": 21453, "bag": 21454, "\u0120Echo": 21455, "\u0120Camb": 21456, "\u0120Madd": 21457, "\u0120Boo": 21458, "\u0120rede": 21459, "\u0120Burning": 21460, "\u0120smoothly": 21461, "\u0120Adrian": 21462, "\u0120Vampire": 21463, "\u0120Monsters": 21464, "steam": 21465, "Style": 21466, "Ma": 21467, "rea": 21468, "\u0120Dwar": 21469, "alyst": 21470, "ursor": 21471, "\u0120elimination": 21472, "\u0120crypto": 21473, "cht": 21474, "\u0120Eternal": 21475, "\u00e2\u0122\u00a6]": 21476, "\u0120Sorce": 21477, "Ill": 21478, "NER": 21479, "\u0120uh": 21480, "Conclusion": 21481, "wage": 21482, "\u0120respir": 21483, "\u0120reminis": 21484, "hetical": 21485, "\u0120gy": 21486, "\u0120utilized": 21487, "icidal": 21488, "\u01201900": 21489, "\u0120hunters": 21490, "\u0120Swan": 21491, "\u0120React": 21492, "\u0120visitor": 21493, "\u0120Thanksgiving": 21494, "308": 21495, "Posts": 21496, "\u0120hips": 21497, "1997": 21498, "omers": 21499, "\u0120knocking": 21500, "\u0120Vehicle": 21501, "\u0120til": 21502, "\u0120138": 21503, "\u0120mi": 21504, "\u0120Investigation": 21505, "\u0120Kenya": 21506, "\u0120casino": 21507, "\u0120motives": 21508, "\u0120regain": 21509, "rex": 21510, "\u0120weekends": 21511, "\u0120stabbed": 21512, "boro": 21513, "\u0120exploited": 21514, "\u0120HAVE": 21515, "\u0120Television": 21516, "cock": 21517, "\u0120preparations": 21518, "\u0120endeav": 21519, "\u0120Remote": 21520, "\u0120Maker": 21521, "\u0120Produ": 21522, "\u0120Evan": 21523, "\u0120informational": 21524, "\u0120Louisville": 21525, "154": 21526, "\u0120Dreams": 21527, "\u0120plots": 21528, "\u0120Runner": 21529, "\u0120hurting": 21530, "\u0120academy": 21531, "\u0120Montgomery": 21532, "nm": 21533, "\u0120Lanc": 21534, "\u0120Alz": 21535, "210": 21536, "elong": 21537, "\u0120retailer": 21538, "\u0120arising": 21539, "\u0120rebellion": 21540, "\u0120blonde": 21541, "played": 21542, "\u0120instrumental": 21543, "Cross": 21544, "\u0120retention": 21545, "\u0120therapeutic": 21546, "\u0120seas": 21547, "\u0120infantry": 21548, "\u0120Clint": 21549, "\u0120prompting": 21550, "\u0120bitch": 21551, "\u0120stems": 21552, "\u0120Kra": 21553, "\u0120thesis": 21554, "\u0120Bog": 21555, "rued": 21556, "\u0120kings": 21557, "\u0120clay": 21558, "ificent": 21559, "\u0120YES": 21560, "\u0120Thing": 21561, "\u0120Cubs": 21562, "veyard": 21563, "elsh": 21564, "inarily": 21565, "\u0120Ey": 21566, "\u0120Rolling": 21567, "\u0120evolving": 21568, "India": 21569, "\u0120recognizes": 21570, "\u0120graduation": 21571, "isers": 21572, "\u0120fertility": 21573, "\u0120Milan": 21574, "Command": 21575, "\u0120boxing": 21576, "\u01201943": 21577, "\u0120gluten": 21578, "\u0120Emir": 21579, "\u0120idol": 21580, "\u0120conceived": 21581, "\u0120Creation": 21582, "Merit": 21583, "uddy": 21584, "ussions": 21585, "\u0120Lieutenant": 21586, "ietal": 21587, "\u0120unchanged": 21588, "\u0120Scale": 21589, "\u0120Crimea": 21590, "balls": 21591, "atorial": 21592, "\u0120depths": 21593, "\u0120empirical": 21594, "\u0120transm": 21595, "\u0120unsafe": 21596, "missible": 21597, "comfort": 21598, "156": 21599, "\u0120mechanic": 21600, "002": 21601, "lins": 21602, "\u0120smoked": 21603, "Pos": 21604, "\u0120slowing": 21605, "\u0120lav": 21606, "Texas": 21607, "\u0120cheating": 21608, "\u0120Metropolitan": 21609, "ethyl": 21610, "\u0120discovering": 21611, "asse": 21612, "\u0120pencil": 21613, "\u0120Pyongyang": 21614, "\u0120closet": 21615, "\u0120Sheet": 21616, "\u0120Entry": 21617, "oustic": 21618, "\u0120myst": 21619, "erate": 21620, "ariat": 21621, "\u0120minerals": 21622, "\u0120musician": 21623, "\u0120Pul": 21624, "\u0120Maz": 21625, "249": 21626, "\u0120permissions": 21627, "\u0120iv": 21628, "enary": 21629, "ickers": 21630, "\u0120Bing": 21631, "hea": 21632, "enable": 21633, "\u0120griev": 21634, "\u0120asserted": 21635, "\u0120Colonel": 21636, "\u0120affidav": 21637, "wo": 21638, "\u0120seated": 21639, "\u0120Ride": 21640, "\u0120paintings": 21641, "\u0120Pix": 21642, "\u0120137": 21643, "ishi": 21644, "umbai": 21645, "gotten": 21646, "\u0120Earl": 21647, "\u0120inning": 21648, "\u0120census": 21649, "\u0120travelled": 21650, "\u0120Consult": 21651, "185": 21652, "bind": 21653, "\u0120simplicity": 21654, "\u0120overlooked": 21655, "\u0120Helpful": 21656, "\u0120monkey": 21657, "\u0120overwhelmingly": 21658, "Blood": 21659, "\u0120Flint": 21660, "\u0120Jama": 21661, "\u0120Present": 21662, "\u0120Rage": 21663, "\u0120TA": 21664, "ptive": 21665, "\u0120turnout": 21666, "wald": 21667, "\u0120Dolphins": 21668, "\u0120VPN": 21669, "\u0120onion": 21670, "\u0120crafting": 21671, "mma": 21672, "\u0120Mercury": 21673, "\u0120arrange": 21674, "\u0120alerts": 21675, "\u0120OT": 21676, "zbollah": 21677, "\u0120gases": 21678, "\u0120Richardson": 21679, "sal": 21680, "lar": 21681, "\u0120frost": 21682, "\u0120lowering": 21683, "\u0120acclaim": 21684, "\u0120startups": 21685, "\u0120Gain": 21686, "essment": 21687, "\u0120guardian": 21688, "\u00e4\u00ba\u00ba": 21689, "\u0120Pie": 21690, "\u0120Links": 21691, "\u0120merits": 21692, "\u0120awake": 21693, "\u0120parental": 21694, "\u0120exceeds": 21695, "\u0120idle": 21696, "\u0120Pilot": 21697, "\u0120eBay": 21698, "\u0120Accept": 21699, "ipeg": 21700, "Cam": 21701, "\u0120Kot": 21702, "\u0120traders": 21703, "olitics": 21704, "unker": 21705, "\u0120Pale": 21706, "osi": 21707, "anmar": 21708, "\u01201947": 21709, "\u0120Fell": 21710, "estial": 21711, "itating": 21712, "GF": 21713, "\u0120Sr": 21714, "ifted": 21715, "\u0120connector": 21716, "\u0120Bone": 21717, "illes": 21718, "260": 21719, "hma": 21720, "\u0120overlap": 21721, "\u0120GitHub": 21722, "\u0120cleaner": 21723, "\u0120Baptist": 21724, "\u0120WAS": 21725, "\u0120lungs": 21726, "\u00d1\u0123": 21727, "\u0120BUT": 21728, "\u0120cite": 21729, "\u0120pitched": 21730, "reatment": 21731, "\u0120trophies": 21732, "\u0120Nu": 21733, "386": 21734, "\u0120Pride": 21735, "\u0120attendees": 21736, "[]": 21737, "179": 21738, "\u0120spatial": 21739, "\u0120prizes": 21740, "\u0120Religion": 21741, "\u0120showcase": 21742, "\u0120Category": 21743, "vidia": 21744, "Target": 21745, "Property": 21746, "?,": 21747, "\u0120fusion": 21748, "pie": 21749, "\u0120UCLA": 21750, "\u0120soundtrack": 21751, "\u0120princess": 21752, "\u0120Caval": 21753, "should": 21754, "\u0120limbs": 21755, "Background": 21756, "\u0120lonely": 21757, "\u0120cores": 21758, "\u0120Tail": 21759, "sheet": 21760, "\u0120132": 21761, "Ra": 21762, "\u00e3\u0124\u00ab": 21763, "\u0120Bolt": 21764, "\u0120booked": 21765, "\u0120administer": 21766, "\u0120equals": 21767, "wy": 21768, "\u0120observing": 21769, "\u0120Baron": 21770, "\u0120Adobe": 21771, "\u0120virgin": 21772, "\u0120Socialist": 21773, "Move": 21774, "ghazi": 21775, "\u0120Linda": 21776, "212": 21777, "\u0120brewing": 21778, "\u0120merchants": 21779, "burse": 21780, "\u0120divor": 21781, "\u0120metals": 21782, "\u0120Ner": 21783, "\u0120sums": 21784, "\u0120Enemy": 21785, "\u0120envision": 21786, "\u0120granting": 21787, "\u0120Honey": 21788, "\u0120Skyrim": 21789, "\u0120socio": 21790, "graded": 21791, "\u0120selective": 21792, "WASHINGTON": 21793, "\u01201948": 21794, "\u0120Sirius": 21795, "\u0120Gross": 21796, "activity": 21797, "\u0120Ivan": 21798, "\u0120furious": 21799, "BSD": 21800, "\u0120Previous": 21801, "\u0120responsive": 21802, "\u0120charitable": 21803, "\u0120leaning": 21804, "\u0120Pew": 21805, "\u0120violates": 21806, "\\\\\\\\\\\\\\\\": 21807, "\u0120Coming": 21808, "wire": 21809, "\u0120poet": 21810, "\u0120resolutions": 21811, "command": 21812, "\u0120Portuguese": 21813, "\u0120nickname": 21814, "\u0120deaf": 21815, "February": 21816, "\u0120recognise": 21817, "\u0120entirety": 21818, "\u0120seasonal": 21819, "placed": 21820, "\u0120Telegraph": 21821, "\u0120microphone": 21822, "ouring": 21823, "\u0120grains": 21824, "\u0120governed": 21825, "\u0120postp": 21826, "\u0120Waters": 21827, "inement": 21828, "\u0120undocumented": 21829, "\u0120Comcast": 21830, "\u0120fox": 21831, "\u0120assaults": 21832, "reon": 21833, "many": 21834, "\u0120Jenkins": 21835, "\u0120Anyway": 21836, "\u0120assessments": 21837, "\u0120downs": 21838, "\u0120Mouse": 21839, "\u0120superb": 21840, "kt": 21841, "\u0120Dow": 21842, "\u0120taxation": 21843, "401": 21844, "\u0120smiles": 21845, "\u0120undertaken": 21846, "\u0120exh": 21847, "\u0120enthusiastic": 21848, "\u0120twent": 21849, "\u0120governmental": 21850, "\u0120autonomy": 21851, "\u0120Technologies": 21852, "\u0120Chain": 21853, "\u0120prevalent": 21854, "fb": 21855, "\u0120nicotine": 21856, "ogram": 21857, "job": 21858, "\u0120awaiting": 21859, "\u0120Menu": 21860, "\u0120deputies": 21861, "kov": 21862, "ishops": 21863, "Button": 21864, "\u0120Shanghai": 21865, "\u0120diesel": 21866, "\u0120Duck": 21867, "Ryan": 21868, "\u0120PCs": 21869, "NF": 21870, "jury": 21871, "ente": 21872, "\u0120inaccurate": 21873, "eddy": 21874, "Whatever": 21875, "\u0120showc": 21876, "\u0120Nad": 21877, "odus": 21878, "etr": 21879, "\u0120plaintiffs": 21880, "\u0120WOR": 21881, "\u0120Assange": 21882, "\u0120privat": 21883, "\u0120premiums": 21884, "\u0120tam": 21885, "URL": 21886, "\u0120elites": 21887, "\u0120Ranger": 21888, "ottenham": 21889, "\u0120Hoff": 21890, "\u0120Athens": 21891, "\u0120definite": 21892, "\u0120sighed": 21893, "\u0120evenly": 21894, "211": 21895, "\u0120Amber": 21896, "akia": 21897, "\u0120mailing": 21898, "\u0120crashing": 21899, "\u0120Confederate": 21900, "rugged": 21901, "Wal": 21902, "\u0120Depths": 21903, "\u0120juvenile": 21904, "\u0120reactor": 21905, "Introduction": 21906, "\u0120Deluxe": 21907, "1995": 21908, "\u0120Sanchez": 21909, "\u0120Mead": 21910, "ivable": 21911, ":-": 21912, "\u0120Planning": 21913, "\u0120Trap": 21914, "quin": 21915, "\u0120Protect": 21916, "vered": 21917, "Information": 21918, "\u0120kidney": 21919, "innamon": 21920, "las": 21921, "\u0120policing": 21922, "\u0120tolerate": 21923, "\u0120Qi": 21924, "\u0120biased": 21925, "Fort": 21926, "\u0120Ki": 21927, "save": 21928, "\u0120privileged": 21929, "\u0120beasts": 21930, "\u0120Glas": 21931, "\u0120Cinem": 21932, "\u0120comeback": 21933, "Sunday": 21934, "\u0120extinction": 21935, "hops": 21936, "\u0120transmit": 21937, "\u0120doubles": 21938, "\u0120Flat": 21939, "167": 21940, "\u0120disputed": 21941, "\u0120injustice": 21942, "foo": 21943, "Vict": 21944, "roleum": 21945, "\u0120Julie": 21946, "Context": 21947, "\u0120Rarity": 21948, "issue": 21949, "Component": 21950, "\u0120counseling": 21951, "anne": 21952, "dark": 21953, "\u0120objections": 21954, "uilt": 21955, "\u0120gast": 21956, "\u0120plac": 21957, "\u0120unused": 21958, "\u00e3\u0125\u0129": 21959, "\u0120Trial": 21960, "\u0120Jas": 21961, "hedral": 21962, "obb": 21963, "\u0120temporal": 21964, "\u0120PRO": 21965, "\u0120NW": 21966, "\u0120Anniversary": 21967, "Large": 21968, "\u0120therm": 21969, "\u0120david": 21970, "\u0120systemic": 21971, "\u0120Shir": 21972, "mut": 21973, "\u0120Nept": 21974, "address": 21975, "\u0120scanning": 21976, "\u0120understandable": 21977, "\u0120canvas": 21978, "Cat": 21979, "\u0120Zoo": 21980, "\u0120angels": 21981, "LO": 21982, "\u0120Statement": 21983, "\u0120Sig": 21984, "ovable": 21985, "\u0120Away": 21986, "sharing": 21987, "ocrats": 21988, "stated": 21989, "\u0120weighing": 21990, "Nor": 21991, "wild": 21992, "Bey": 21993, "\u0120astonishing": 21994, "\u0120Reynolds": 21995, "\u0120opener": 21996, "\u0120trainer": 21997, "\u0120surgical": 21998, "pn": 21999, "\u0120adjusting": 22000, "wheel": 22001, "\u0120frown": 22002, "ervative": 22003, "\u0120suspend": 22004, "Within": 22005, "tein": 22006, "\u0120obstacle": 22007, "\u0120liberties": 22008, "ymes": 22009, "\u0120uranium": 22010, "ansom": 22011, "anol": 22012, "uba": 22013, "\u0120Loss": 22014, "\u0120arous": 22015, "\u0120Henderson": 22016, "Wow": 22017, "spl": 22018, "cur": 22019, "\u0120\u00c2\u0143": 22020, "\u0120theirs": 22021, "Damage": 22022, "\u0120downloading": 22023, "\u0120discern": 22024, "\u0120Sto": 22025, "\u0120Fla": 22026, "\u0120hath": 22027, "\u0120Aj": 22028, "\u0120unpleasant": 22029, "European": 22030, "expensive": 22031, "\u0120screenshot": 22032, "\u0120UV": 22033, "\u0120allied": 22034, "\u0120Persian": 22035, "\u0120monopoly": 22036, "\u0120atom": 22037, "\u0120Redskins": 22038, "\"><": 22039, "\u0120cancell": 22040, "\u0120cinema": 22041, "131": 22042, "fair": 22043, "\u0120Alfred": 22044, "\u0120duck": 22045, "args": 22046, "223": 22047, "\u0120ISI": 22048, "\u0120signaling": 22049, "inar": 22050, "\u0120laughs": 22051, "\u0120forwards": 22052, "\u0120reckless": 22053, "\u0120listeners": 22054, "ativity": 22055, "\u0120vastly": 22056, "nant": 22057, "Less": 22058, "\u0120Hunting": 22059, "\u0120Scientific": 22060, "ITED": 22061, "\u0120knight": 22062, "\u0120HTC": 22063, "usa": 22064, "tmp": 22065, "\u0120rude": 22066, "\u0120Legendary": 22067, "\u0120arises": 22068, "Bad": 22069, "\u0120Claim": 22070, "peg": 22071, "\u0120realities": 22072, "Think": 22073, "\u0120\u00c2\u00b0": 22074, "\u0120rode": 22075, "\u0120strive": 22076, "\u0120anecd": 22077, "\u0120shorts": 22078, "\u0120hypothes": 22079, "\u0120coordinated": 22080, "\u0120Gandhi": 22081, "\u0120FPS": 22082, "RED": 22083, "\u0120susceptible": 22084, "\u0120shrink": 22085, "\u0120Chart": 22086, "Help": 22087, "\u0120ion": 22088, "deep": 22089, "ribes": 22090, "\u0120Kai": 22091, "\u0120Customer": 22092, "Summary": 22093, "\u0120cough": 22094, "wife": 22095, "\u0120lend": 22096, "\u0120positioning": 22097, "\u0120lottery": 22098, "\u0120Canyon": 22099, "\u0120fade": 22100, "\u0120bronze": 22101, "\u0120Kenny": 22102, "\u0120boasts": 22103, "\u0120Enhanced": 22104, "record": 22105, "\u0120emergence": 22106, "\u0120akin": 22107, "\u0120Bert": 22108, "itous": 22109, "\u00e2\u0138\u0133": 22110, "\u0120stip": 22111, "\u0120exchanged": 22112, "omore": 22113, "alsh": 22114, "\u0120reservoir": 22115, "\u0120standpoint": 22116, "WM": 22117, "\u0120initiate": 22118, "\u0120decay": 22119, "\u0120brewery": 22120, "\u0120terribly": 22121, "\u0120mortal": 22122, "levard": 22123, "\u0120revis": 22124, "NI": 22125, "elo": 22126, "\u0120confess": 22127, "\u0120MSNBC": 22128, "\u0120submissions": 22129, "Controller": 22130, "\u0120202": 22131, "\u0120Ruth": 22132, "});": 22133, "\u0120Azure": 22134, "\u0120.\"": 22135, "206": 22136, "\u0120Marketing": 22137, "\u0120laund": 22138, "iencies": 22139, "\u0120renowned": 22140, "\u0120Trou": 22141, "\u0120NGO": 22142, "blems": 22143, "\u0120terrified": 22144, "\u0120warns": 22145, "\u0120pert": 22146, "\u0120unsure": 22147, "480": 22148, "alez": 22149, "ultz": 22150, "\u0120Outside": 22151, "\u0120styl": 22152, "\u0120Underground": 22153, "\u0120panc": 22154, "\u0120dictionary": 22155, "\u0120foe": 22156, "riminal": 22157, "\u0120Norwegian": 22158, "\u0120jailed": 22159, "\u0120maternal": 22160, "\u00c3\u00a9e": 22161, "\u0120Lucy": 22162, "cop": 22163, "Cho": 22164, "\u0120unsigned": 22165, "\u0120Zelda": 22166, "\u0120Insider": 22167, "\u0120Continued": 22168, "\u0120133": 22169, "\u0120Naruto": 22170, "\u0120Majority": 22171, "169": 22172, "\u0120Wo": 22173, "\u00e3\u0124\u0135": 22174, "\u0120pastor": 22175, "\u0120informal": 22176, "\u00d0\u00bd": 22177, "anthrop": 22178, "join": 22179, "\u00e3\u0123\u0139": 22180, "itational": 22181, "NP": 22182, "\u0120Writing": 22183, "fn": 22184, "\u0120Bever": 22185, "195": 22186, "\u0120yelling": 22187, "\u0120drastically": 22188, "\u0120eject": 22189, "\u0120neut": 22190, "\u0120thrive": 22191, "\u0120Frequ": 22192, "oux": 22193, "\u0120possesses": 22194, "\u0120Senators": 22195, "\u0120DES": 22196, "\u0120Shakespeare": 22197, "\u0120Franco": 22198, "\u0120LB": 22199, "uchi": 22200, "\u0120incarn": 22201, "\u0120founders": 22202, "Function": 22203, "\u0120brightness": 22204, "\u0120BT": 22205, "\u0120whale": 22206, "\u0120Theater": 22207, "mass": 22208, "\u0120Doll": 22209, "Something": 22210, "\u0120echoed": 22211, "\u0120Hex": 22212, "crit": 22213, "afia": 22214, "\u0120goddess": 22215, "\u0120eleven": 22216, "\u0120Preview": 22217, "\u0120Aurora": 22218, "\u0120401": 22219, "ulsive": 22220, "\u0120Logan": 22221, "inburgh": 22222, "\u0120Centers": 22223, "\u0120ONLY": 22224, "\u0120Aid": 22225, "\u0120paradox": 22226, "\u0120hurd": 22227, "\u0120LC": 22228, "Due": 22229, "court": 22230, "\u0120offended": 22231, "\u0120evaluating": 22232, "\u0120Matthews": 22233, "\u0120tomb": 22234, "\u0120payroll": 22235, "\u0120extraction": 22236, "\u0120Hands": 22237, "ifi": 22238, "\u0120supernatural": 22239, "\u0120COMM": 22240, "]=": 22241, "dogs": 22242, "\u0120512": 22243, "\u0120Meeting": 22244, "Richard": 22245, "\u0120Maximum": 22246, "\u0120ideals": 22247, "Things": 22248, "mand": 22249, "\u0120Regardless": 22250, "\u0120humili": 22251, "buffer": 22252, "Little": 22253, "\u0120Dani": 22254, "\u0120Nak": 22255, "\u0120liberation": 22256, "\u0120Abe": 22257, "\u0120OL": 22258, "\u0120stuffed": 22259, "aca": 22260, "inda": 22261, "raphic": 22262, "\u0120mosqu": 22263, "\u0120campaigning": 22264, "\u0120occupy": 22265, "Squ": 22266, "rina": 22267, "\u0120Wel": 22268, "\u0120VS": 22269, "\u0120physic": 22270, "\u0120puls": 22271, "rint": 22272, "oaded": 22273, "ETF": 22274, "\u0120Archives": 22275, "\u0120venues": 22276, "hner": 22277, "\u0120Turbo": 22278, "\u0120lust": 22279, "\u0120appealed": 22280, "quez": 22281, "ilib": 22282, "\u0120Timothy": 22283, "\u0120omn": 22284, "dro": 22285, "\u0120obsession": 22286, "\u0120Savage": 22287, "1996": 22288, "Global": 22289, "Jes": 22290, "214": 22291, "\u0120sliding": 22292, "\u0120disappro": 22293, "\u0120Magical": 22294, "\u0120voluntarily": 22295, "gb": 22296, "aney": 22297, "\u0120prophet": 22298, "\u0120Rein": 22299, "\u0120Julia": 22300, "\u0120Worth": 22301, "aurus": 22302, "\u0120bounds": 22303, "ieu": 22304, ")))": 22305, "\u0120crore": 22306, "\u0120Citizen": 22307, "Sky": 22308, "\u0120columnist": 22309, "\u0120seekers": 22310, "ondo": 22311, "ISA": 22312, "\u0120Length": 22313, "\u0120nostalg": 22314, "\u0120newcom": 22315, "\u0120detrim": 22316, "entric": 22317, "375": 22318, "\u0120GE": 22319, "\u0120autop": 22320, "\u0120academics": 22321, "AppData": 22322, "\u0120Shen": 22323, "\u0120idiot": 22324, "\u0120Transit": 22325, "\u0120teaspoon": 22326, "Wil": 22327, "KO": 22328, "\u0120Comedy": 22329, ">,": 22330, "\u0120populated": 22331, "WD": 22332, "\u0120pigs": 22333, "\u0120Oculus": 22334, "\u0120sympathetic": 22335, "\u0120marathon": 22336, "198": 22337, "\u0120seizure": 22338, "sided": 22339, "\u0120dop": 22340, "irtual": 22341, "Land": 22342, "\u0120Floor": 22343, "osaurs": 22344, "...]": 22345, "\u0120los": 22346, "\u0120subsidiary": 22347, "EY": 22348, "\u0120Parts": 22349, "\u0120Stef": 22350, "\u0120Judiciary": 22351, "\u0120134": 22352, "\u0120mirrors": 22353, "\u0120ket": 22354, "times": 22355, "\u0120neurolog": 22356, "\u0120cav": 22357, "\u0120Guest": 22358, "\u0120tumor": 22359, "scill": 22360, "\u0120Lloyd": 22361, "Est": 22362, "\u0120clearer": 22363, "\u0120stereotypes": 22364, "\u0120dur": 22365, "nothing": 22366, "Reddit": 22367, "\u0120negotiated": 22368, "------------------------": 22369, "235": 22370, "\u0120flown": 22371, "\u0120Seoul": 22372, "\u0120Resident": 22373, "\u0120SCH": 22374, "\u0120disappearance": 22375, "\u0120Vince": 22376, "grown": 22377, "\u0120grabs": 22378, "ril": 22379, "\u0120Infinite": 22380, "\u0120Twenty": 22381, "\u0120pedestrian": 22382, "\u0120jersey": 22383, "\u0120Fur": 22384, "\u0120Infinity": 22385, "\u0120Elliott": 22386, "\u0120mentor": 22387, "\u0120morally": 22388, "\u0120obey": 22389, "secure": 22390, "iffe": 22391, "\u0120antibiotics": 22392, "angled": 22393, "\u0120Freeman": 22394, "\u0120Introduction": 22395, "Jun": 22396, "\u0120marsh": 22397, "icans": 22398, "\u0120EVENTS": 22399, "ochond": 22400, "Wall": 22401, "iculty": 22402, "\u0120misdemeanor": 22403, "\u0120ly": 22404, "Thomas": 22405, "\u0120Resolution": 22406, "\u0120animations": 22407, "\u0120Dry": 22408, "\u0120intercourse": 22409, "\u0120Newcastle": 22410, "\u0120Hog": 22411, "\u0120Equipment": 22412, "177": 22413, "\u0120territorial": 22414, "\u0120archives": 22415, "203": 22416, "Filter": 22417, "\u0120Munich": 22418, "\u0120commanded": 22419, "\u0120Wand": 22420, "\u0120pitches": 22421, "\u0120Croat": 22422, "\u0120ratios": 22423, "\u0120Mits": 22424, "\u0120accumulated": 22425, "\u0120Specifically": 22426, "\u0120gentleman": 22427, "acerb": 22428, "\u0120penn": 22429, "\u0120aka": 22430, "\u0120Fuk": 22431, "\u0120intervene": 22432, "\u0120Refuge": 22433, "\u0120Alzheimer": 22434, "\u0120succession": 22435, "ohan": 22436, "does": 22437, "Lord": 22438, "\u0120separat": 22439, "\u0120correspondence": 22440, "\u0120shiny": 22441, "Prior": 22442, "\u0120sulf": 22443, "\u0120miserable": 22444, "\u0120dedication": 22445, "().": 22446, "\u0120specialists": 22447, "\u0120defects": 22448, "\u0120Cult": 22449, "\u0120Xia": 22450, "\u0120jeopard": 22451, "\u0120Ore": 22452, "Ability": 22453, "\u0120lear": 22454, "\u0120ambitions": 22455, "\u0120BMI": 22456, "\u0120Arabs": 22457, "\u01201942": 22458, "\u0120preservation": 22459, "ificate": 22460, "\u0120ashamed": 22461, "loss": 22462, "\u0120Restaur": 22463, "\u0120resemble": 22464, "\u0120enrich": 22465, "\u0120KN": 22466, "\u0120Clan": 22467, "float": 22468, "\u0120playable": 22469, "ITT": 22470, "\u0120harmony": 22471, "arrison": 22472, "\u0120Weinstein": 22473, "were": 22474, "\u0120poisoning": 22475, "\u0120Comput": 22476, "\u0120WordPress": 22477, "major": 22478, "\u0120Valve": 22479, "Fan": 22480, "\u0120Throw": 22481, "\u0120Romans": 22482, "\u0120Depression": 22483, "ados": 22484, "\u0120tortured": 22485, "\u0120balancing": 22486, "bottom": 22487, "\u0120acquiring": 22488, "\u0120Monte": 22489, "ardi": 22490, "\u0120aura": 22491, "\u0120##": 22492, "\u0120Standing": 22493, "\u0120Atlas": 22494, "CF": 22495, "\u0120intrins": 22496, "\u0120Benghazi": 22497, "\u0120camping": 22498, "\u0120tapped": 22499, "blade": 22500, "strous": 22501, "\u0120Rabb": 22502, "\u0120Written": 22503, "tip": 22504, "\u0120Neigh": 22505, "sterdam": 22506, "\u0120Allow": 22507, "\u0120Healing": 22508, "\u0120Rhod": 22509, "num": 22510, "\u0120caffeine": 22511, "\u0120Percent": 22512, "\u0120boo": 22513, "\u0120apples": 22514, "305": 22515, "\u0120welcoming": 22516, "\u0120applaud": 22517, "\u0120austerity": 22518, "\u00c2\u00b1": 22519, "\u0120Reality": 22520, "efe": 22521, "\u00e5\u00ae": 22522, "\u0120sucks": 22523, "\u0120tabs": 22524, "\u0120PayPal": 22525, "\u0120backpack": 22526, "\u0120gifted": 22527, "abulary": 22528, "\u0120Scout": 22529, "irteen": 22530, "\u0120chin": 22531, "\u0120omitted": 22532, "\u0120negatively": 22533, "\u0120accessing": 22534, "\u0120Earn": 22535, "\u0120ambulance": 22536, "\u0120headphones": 22537, "\u0120205": 22538, "\u0120Refresh": 22539, "president": 22540, "\u0120Kitchen": 22541, "\u0120Entered": 22542, "\u0120Snyder": 22543, "005": 22544, "omical": 22545, "\u0120borrowed": 22546, "\u0120Nem": 22547, "\u0120aviation": 22548, "\u0120stall": 22549, "rimination": 22550, "\u0120uniforms": 22551, "itime": 22552, "\u0120Simmons": 22553, "energy": 22554, "ablished": 22555, "yy": 22556, "qualified": 22557, "\u0120rallies": 22558, "\u0120Stuart": 22559, "flight": 22560, "\u0120gangs": 22561, "rag": 22562, "\u0120vault": 22563, "lux": 22564, "\u0120Compar": 22565, "\u0120designation": 22566, "209": 22567, "\u0120Jos": 22568, "dollar": 22569, "zero": 22570, "\u0120wells": 22571, "303": 22572, "\u0120constituents": 22573, "\u0120heck": 22574, "\u0120cows": 22575, "\u0120commanders": 22576, "\u0120differential": 22577, "\u0120Catherine": 22578, "299": 22579, "\u0120valve": 22580, "\u0120brace": 22581, "\u0120perspectives": 22582, "cert": 22583, "fact": 22584, "icularly": 22585, "\u0120McN": 22586, "planes": 22587, "\u0120intric": 22588, "\u0120peas": 22589, "ovan": 22590, "\u0120tossed": 22591, "retch": 22592, "\u0120Lopez": 22593, "\u0120unfamiliar": 22594, "death": 22595, "\u0120Apart": 22596, "\u0120Chang": 22597, "\u0120relieved": 22598, "rophe": 22599, "\u0120airports": 22600, "\u0120freak": 22601, "util": 22602, "Mill": 22603, "\u0120Chin": 22604, "\u0120Owen": 22605, "male": 22606, "\u0120Broken": 22607, "\u0120Winds": 22608, "rob": 22609, "rising": 22610, "\u0120firefighters": 22611, "\u0120authoritarian": 22612, "\u0120148": 22613, "Bitcoin": 22614, "external": 22615, "\u0120browsers": 22616, "ichever": 22617, "orian": 22618, "\u0120unb": 22619, "\u0120poke": 22620, "\u0120Zot": 22621, "Mid": 22622, "\u0120Popular": 22623, "\u0120covert": 22624, "\u0120contributes": 22625, "\u0120650": 22626, "\u0120contention": 22627, "Gate": 22628, "\u0120consoles": 22629, "\u0120chromos": 22630, "\u0120IX": 22631, "\u0120visually": 22632, "\u0120Eisen": 22633, "\u0120jewelry": 22634, "\u0120delegation": 22635, "\u0120accelerate": 22636, "\u0120Riley": 22637, "\u0120slope": 22638, "\u0120indoor": 22639, "itially": 22640, "\u0120hugely": 22641, "\u0120tunnels": 22642, "\u0120fined": 22643, "\u0120directive": 22644, "\u0120forehead": 22645, "ustomed": 22646, "\u0120skate": 22647, "Music": 22648, "gas": 22649, "\u0120recognizing": 22650, "ambo": 22651, "\u0120overweight": 22652, "\u0120Grade": 22653, "\u00d9\u012c": 22654, "\u0120sounding": 22655, "\u0120locking": 22656, "\u0120REM": 22657, "Store": 22658, "\u0120excav": 22659, "\u0120Likewise": 22660, "\u0120Lights": 22661, "\u0120elbow": 22662, "\u0120Supply": 22663, "wic": 22664, "\u0120handsome": 22665, "1994": 22666, "Coll": 22667, "\u0120adequately": 22668, "\u0120Associate": 22669, "\u0120strips": 22670, "\u0120crackdown": 22671, "\u0120marvel": 22672, "\u0120Kun": 22673, "\u0120passages": 22674, "@@@@": 22675, "\u0120Tall": 22676, "\u0120thoughtful": 22677, "namese": 22678, "\u0120prostitution": 22679, "business": 22680, "\u0120ballistic": 22681, "personal": 22682, "cig": 22683, "izational": 22684, "Round": 22685, "\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142": 22686, "\u0120Coleman": 22687, "\u0120admitting": 22688, "\u0120Plug": 22689, "\u0120bitcoins": 22690, "\u0120Suz": 22691, "\u0120fairness": 22692, "\u0120supplier": 22693, "\u0120catastrophic": 22694, "\u0120Helen": 22695, "oqu": 22696, "Marc": 22697, "\u0120Articles": 22698, "gie": 22699, "\u0120endangered": 22700, "\u0120destiny": 22701, "\u0120Volt": 22702, "olia": 22703, "axis": 22704, "\u0120cheat": 22705, "\u0120unified": 22706, "ICO": 22707, "quote": 22708, "302": 22709, "\u0120Sed": 22710, "\u0120suppression": 22711, "\u0120analyzing": 22712, "\u0120squat": 22713, "\u0120figuring": 22714, "\u0120coordinates": 22715, "\u0120chunks": 22716, "\u01201946": 22717, "\u0120subp": 22718, "\u0120wiki": 22719, "\u0120Forbes": 22720, "\u0120Jupiter": 22721, "\u0120Erik": 22722, "imer": 22723, "\u0120Commercial": 22724, "\\)": 22725, "\u0120legitimacy": 22726, "\u0120dental": 22727, "\u0120Mean": 22728, "\u0120deficits": 22729, "550": 22730, "Originally": 22731, "\u0120Horror": 22732, "\u0120contamination": 22733, "llah": 22734, "\u0120confisc": 22735, "\u0120Clare": 22736, "TB": 22737, "\u0120Failed": 22738, "aned": 22739, "\u0120ruler": 22740, "\u0120Controller": 22741, "\u0120feminists": 22742, "Fix": 22743, "gay": 22744, "207": 22745, "\u0120rabbit": 22746, "Third": 22747, "owntown": 22748, "\u0120glue": 22749, "\u0120volatile": 22750, "\u0120shining": 22751, "\u0120foll": 22752, "\u0120impaired": 22753, "\u0120supers": 22754, "\u00e6\u012a": 22755, "\u0120clutch": 22756, "\u013c\u00e9\u0128\u0134": 22757, "\u0120prolet": 22758, "\u0120(!": 22759, "\u0120yelled": 22760, "\u0120Kiev": 22761, "\u0120Ern": 22762, "\u0120Shock": 22763, "KB": 22764, "\u0120situated": 22765, "query": 22766, "\u0120Nas": 22767, "\u0120annex": 22768, "character": 22769, "\u0120Holiday": 22770, "\u0120automation": 22771, "\u0120Jill": 22772, "\u0120Remastered": 22773, "\u0120linem": 22774, "\u0120wilderness": 22775, "\u0120Horizon": 22776, "\u0120Guinea": 22777, "AZ": 22778, "\u0120mainland": 22779, "\u0120secrecy": 22780, "LEASE": 22781, "\u0120punk": 22782, "\u0120Province": 22783, "(),": 22784, "Speed": 22785, "\u0120handing": 22786, "\u0120Sebast": 22787, "Sir": 22788, "rase": 22789, "\u0120journals": 22790, "\u0120congest": 22791, "\u0120Tut": 22792, "irrel": 22793, "\u0120schizophrenia": 22794, "\u0120misogyn": 22795, "healthy": 22796, "Iron": 22797, "\u0120reacted": 22798, "-$": 22799, "252": 22800, "\u0120plural": 22801, "\u0120plum": 22802, "\u0120bargain": 22803, "\u0120grounded": 22804, "finder": 22805, "\u0120disse": 22806, "\u0120Laz": 22807, "OOD": 22808, "\u0120atroc": 22809, "Factory": 22810, "\u0120minions": 22811, "\u0120ori": 22812, "\u0120Brave": 22813, "\u0120PRE": 22814, "\u0120Myanmar": 22815, "\u0120Hod": 22816, "\u0120expedition": 22817, "\u0120explode": 22818, "\u0120Coord": 22819, "\u0120extr": 22820, "\u0120Brief": 22821, "\u0120ADHD": 22822, "\u0120hardcore": 22823, "feeding": 22824, "\u0120dile": 22825, "\u0120Fruit": 22826, "\u0120vaccination": 22827, "\u0120Mao": 22828, "osphere": 22829, "\u0120contests": 22830, "-|": 22831, "\u0120fren": 22832, "isphere": 22833, "Rom": 22834, "\u0120Sharp": 22835, "\u0120Trend": 22836, "\u0120disconnect": 22837, "\u00e2\u0122\u00a2\u00e2\u0122\u00a2": 22838, "\u0120persecution": 22839, "Earth": 22840, "\u0120healthier": 22841, "384": 22842, "\u0120cob": 22843, "\u0120Trinity": 22844, "OWS": 22845, "ANN": 22846, "\u0120specialty": 22847, "\u0120gru": 22848, "\u0120cooperative": 22849, "why": 22850, "Starting": 22851, "\u0120Issues": 22852, "stre": 22853, "ensor": 22854, "\u0120185": 22855, "Adv": 22856, "!?": 22857, "\u0120Revel": 22858, "emia": 22859, "\u0120Hulk": 22860, "\u0120celebrations": 22861, "\u0120Sou": 22862, "raud": 22863, "\u0120Klein": 22864, "\u0120unreal": 22865, "context": 22866, "\u0120partnerships": 22867, "\u0120adopting": 22868, "tical": 22869, "\u0120splash": 22870, "\u0120Hezbollah": 22871, "category": 22872, "cyclop": 22873, "xton": 22874, "\u0120Dot": 22875, "urdy": 22876, "tz": 22877, "\u0120envelope": 22878, "\u0120NL": 22879, "\u00e2\u0137": 22880, "\u0120wherein": 22881, "Spec": 22882, "184": 22883, "\u0120telev": 22884, "aliation": 22885, "\u0120myths": 22886, "\u00e5\u00b0": 22887, "\u0120rigorous": 22888, "\u0120communicating": 22889, "\u0120observer": 22890, "\u0120rehe": 22891, "\u0120Wash": 22892, "\u0120apologized": 22893, "\u0120Tin": 22894, "\u0120expenditures": 22895, "workers": 22896, "document": 22897, "\u0120hesitate": 22898, "\u0120Lenin": 22899, "\u0120unpredictable": 22900, "\u0120renewal": 22901, "cler": 22902, "okia": 22903, "\u0120CONT": 22904, "\u0120postseason": 22905, "Tokens": 22906, "\u0120exacerb": 22907, "\u0120betting": 22908, "\u0120147": 22909, "\u0120elevation": 22910, "Wood": 22911, "\u0120Solomon": 22912, "194": 22913, "004": 22914, "output": 22915, "\u0120redund": 22916, "\u0120Mumbai": 22917, "\u0120pH": 22918, "\u0120reproduce": 22919, "\u0120Duration": 22920, "MAX": 22921, "\u0120bog": 22922, "CBS": 22923, "\u0120Balance": 22924, "\u0120Sgt": 22925, "\u0120Recent": 22926, "\u0120cd": 22927, "\u0120popped": 22928, "\u0120incompet": 22929, "prop": 22930, "ayan": 22931, "guy": 22932, "Pacific": 22933, "\u0120tyr": 22934, "\u0120{{": 22935, "\u0120Mystic": 22936, "\u0120Dana": 22937, "\u0120masturb": 22938, "\u0120geometry": 22939, "\u00c3\u00a2": 22940, "\u0120Correct": 22941, "\u0120trajectory": 22942, "\u0120distracted": 22943, "\u0120foo": 22944, "\u0120Welsh": 22945, "Luc": 22946, "mith": 22947, "\u0120rugby": 22948, "\u0120respiratory": 22949, "\u0120triangle": 22950, "\u0120215": 22951, "\u0120undergraduate": 22952, "\u0120Superior": 22953, "changing": 22954, "_-": 22955, "\u0120rightly": 22956, "\u0120referee": 22957, "\u0120lucrative": 22958, "\u0120unauthorized": 22959, "\u0120resembles": 22960, "\u0120GNU": 22961, "\u0120Derby": 22962, "\u0120pathways": 22963, "\u0120Led": 22964, "\u0120endurance": 22965, "\u0120stint": 22966, "\u0120collector": 22967, "Fast": 22968, "\u0120dots": 22969, "\u0120nationals": 22970, "\u0120Securities": 22971, "\u0120whip": 22972, "Param": 22973, "\u0120learns": 22974, "Magic": 22975, "\u0120detailing": 22976, "moon": 22977, "\u0120broadcasting": 22978, "\u0120baked": 22979, "265": 22980, "holm": 22981, "\u0120Sah": 22982, "\u0120Hussein": 22983, "\u0120Courtesy": 22984, "174": 22985, "\u0120146": 22986, "\u0120geographic": 22987, "peace": 22988, "\u0120judging": 22989, "\u0120Stern": 22990, "Bur": 22991, "\u0120storyline": 22992, "Gun": 22993, "\u0120Stick": 22994, "245": 22995, "307": 22996, "\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 22997, "\u0120Administrator": 22998, "\u0120burnt": 22999, "\u0120pave": 23000, "choes": 23001, "Exec": 23002, "\u0120campuses": 23003, "Result": 23004, "\u0120mutations": 23005, "\u0120Charter": 23006, "\u0120captures": 23007, "\u0120compares": 23008, "\u0120badge": 23009, "Scient": 23010, "\u0120erad": 23011, "iery": 23012, "oi": 23013, "ettes": 23014, "\u0120Estate": 23015, "\u0120strap": 23016, "\u0120proudly": 23017, "\u0120fried": 23018, "\u0120withdrawn": 23019, "\u0120Voy": 23020, "phony": 23021, "Items": 23022, "\u0120Pierce": 23023, "bard": 23024, "\u0120annotation": 23025, "anton": 23026, "illon": 23027, "Impro": 23028, "...)": 23029, "\u0120happier": 23030, "------": 23031, "adjust": 23032, "\u0120staffers": 23033, "\u0120activism": 23034, "\u0120perf": 23035, "\u0120alright": 23036, "Need": 23037, "\u0120commence": 23038, "\u0120opioid": 23039, "\u0120Amanda": 23040, "Es": 23041, "\u0120Pars": 23042, "\u0120Kaw": 23043, "Works": 23044, "248": 23045, "\u0120indo": 23046, "tc": 23047, "endant": 23048, "\u0120Moto": 23049, "\u0120legalization": 23050, "OTE": 23051, "\u0120tasked": 23052, "\u0120tsp": 23053, "\u0120ACTIONS": 23054, "166": 23055, "\u0120refreshing": 23056, "\u0120NR": 23057, "\u0120Perez": 23058, "\u0120infringement": 23059, "SY": 23060, "Listen": 23061, "inning": 23062, "ku": 23063, "\u0120rotate": 23064, "program": 23065, "arah": 23066, "Design": 23067, "\u0120(\u00c2\u00a3": 23068, "\u0120storing": 23069, "\u0120warrants": 23070, "\u0120judgement": 23071, "\u0120Brist": 23072, "usually": 23073, "photo": 23074, "\u0120Ran": 23075, "\u0120Pine": 23076, "\u0120outrageous": 23077, "\u0120Valentine": 23078, "luence": 23079, "\u0120Everybody": 23080, "Altern": 23081, "\u0120relevance": 23082, "\u0120terminated": 23083, "\u0120dessert": 23084, "\u0120fulfilled": 23085, "\u0120prosecuted": 23086, "\u0120Words": 23087, "\u0120migrant": 23088, "\u0120cultivation": 23089, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 23090, "idelity": 23091, "\u0120Vern": 23092, "\u0120Login": 23093, "\u0120metaphor": 23094, "\u0120Tip": 23095, "\u0120recruits": 23096, "\u0120Pig": 23097, "ribing": 23098, "\u0120enthusiasts": 23099, "exper": 23100, "\u0120frightening": 23101, "\u0120Hair": 23102, "anson": 23103, "strate": 23104, "\u0120hi": 23105, "Height": 23106, "\u0120owning": 23107, "none": 23108, "\u0120dislike": 23109, "\u0120knives": 23110, "pherd": 23111, "\u0120loudly": 23112, "\u0120APIs": 23113, "Display": 23114, "\u0120Lac": 23115, "\u0120USS": 23116, "abl": 23117, "verages": 23118, "Jew": 23119, "\u0120172": 23120, "\u0120Historical": 23121, "atoon": 23122, "\u0120Physics": 23123, "intern": 23124, "\u0120warmth": 23125, "\u0120topp": 23126, "DM": 23127, "\u0120gunman": 23128, "\u0120emperor": 23129, "odi": 23130, "\u00e3\u0125\u00a3": 23131, "inatory": 23132, "\u0120Rib": 23133, "\u0120131": 23134, "\u0120Saturn": 23135, "\u0120Shining": 23136, "\u0120waking": 23137, "Quotes": 23138, "\u0120comedian": 23139, "enberg": 23140, "\u00c2\u00bd": 23141, "\u0120believers": 23142, "\u0120paperwork": 23143, "custom": 23144, "\u0120lev": 23145, "\u0120lament": 23146, "\u0120pouring": 23147, "222": 23148, "political": 23149, "\u0120Supplement": 23150, "maid": 23151, "\u0120cruelty": 23152, "\u0120tread": 23153, "ysics": 23154, "Aw": 23155, "rites": 23156, "\u0120modifier": 23157, "\u0120Position": 23158, "Adam": 23159, "lb": 23160, "ubs": 23161, "\u0120imperfect": 23162, "\u0120clusters": 23163, "\u0120Engineer": 23164, "\u0120Cherry": 23165, "\u0120inauguration": 23166, "\u0120Sau": 23167, "\u0120embodiment": 23168, "\u0120Uncle": 23169, "\u0120overr": 23170, "\u0120explosions": 23171, "cule": 23172, "\u0120Princeton": 23173, "\u0120Andrea": 23174, "\u0120incorrectly": 23175, "\u0120earnest": 23176, "\u0120pilgr": 23177, "\u0120Sprint": 23178, "\u0120sleeve": 23179, "\u0120hears": 23180, "\u0120Amazing": 23181, "\u0120browsing": 23182, "agin": 23183, "\u0120homeland": 23184, "\u0120haw": 23185, "\u0120diving": 23186, "istered": 23187, "178": 23188, "\u0120bargaining": 23189, "\u0120Arcade": 23190, "\u0120delegate": 23191, "terson": 23192, "................................................................": 23193, "\u0120Jacksonville": 23194, "275": 23195, "\u0120stagn": 23196, "\u0120adam": 23197, "\u0120Sherman": 23198, "CB": 23199, "\u0120suburb": 23200, "\u0120Foods": 23201, "\u0120converting": 23202, "\u0120Arist": 23203, "\u0120chambers": 23204, "love": 23205, "\u0120amino": 23206, "\u0120Gan": 23207, "\u0120madness": 23208, "mc": 23209, "\u0120USE": 23210, "defined": 23211, "\u0120ultr": 23212, "indust": 23213, "\u0120wolves": 23214, "lance": 23215, "Additionally": 23216, "\u0120cracks": 23217, "asia": 23218, "\u0120Reason": 23219, "\u0120Pump": 23220, "\u0120accidental": 23221, "\u0120Laser": 23222, "\u0120Rid": 23223, "\u0120initialized": 23224, "elli": 23225, "\u0120unnamed": 23226, "\u0120noun": 23227, "\u0120Passed": 23228, "\u0120hostage": 23229, "\u0120Ethiop": 23230, "shirts": 23231, "\u0120unrel": 23232, "\u0120Embassy": 23233, "\u01201941": 23234, "\u0120atoms": 23235, "\u0120purported": 23236, "164": 23237, "\u0120Fi": 23238, "\u0120gallons": 23239, "\u0120Monica": 23240, "\u0120pg": 23241, "enment": 23242, "\u0120sorted": 23243, "\u0120Gospel": 23244, "\u0120heights": 23245, "\u0120traced": 23246, "\u0120undergoing": 23247, "Shell": 23248, "\u0120sacks": 23249, "\u0120proportions": 23250, "\u0120halluc": 23251, "Font": 23252, "acet": 23253, "\u0120warmer": 23254, "\u0120INTER": 23255, "\u0120grabbing": 23256, "Plug": 23257, "\u0120realization": 23258, "\u0120Burke": 23259, "\u0120enchant": 23260, "ATER": 23261, "\u0120Seed": 23262, "\u0120abundant": 23263, "FM": 23264, "\u0120civic": 23265, "Vs": 23266, "isi": 23267, "\u0120vow": 23268, "\u0120reper": 23269, "\u0120Partnership": 23270, "\u0120penetration": 23271, "\u0120axe": 23272, "\u0120shattered": 23273, "\u0120Zombies": 23274, "\u0120vinyl": 23275, "\u0120Alert": 23276, "eon": 23277, "\u0120obliged": 23278, "\u0120Illust": 23279, "\u0120Plaza": 23280, "\u0120Frontier": 23281, "\u0120davidjl": 23282, "\u0120Serial": 23283, "\u0120Hav": 23284, "\u0120Nutrition": 23285, "Bi": 23286, "\u0120\u00e2\u0138\u012a": 23287, "\u0120Jays": 23288, "linux": 23289, "\u0120hurry": 23290, "\u0120voy": 23291, "\u0120hopeless": 23292, "\u0120Stealth": 23293, "\u0120\u00e3\u0123": 23294, "essors": 23295, "ttle": 23296, "borg": 23297, "\u0120Safari": 23298, "fell": 23299, "\u0120wary": 23300, "due": 23301, "\u0120Above": 23302, "Ha": 23303, "ELL": 23304, "\u0120notor": 23305, "\u0120Won": 23306, "Too": 23307, "\u0120occupations": 23308, "\u0120possessions": 23309, "\u0120inviting": 23310, "\u0120predators": 23311, "\u0120accelerated": 23312, "\u0120157": 23313, "uterte": 23314, "\u0120Cube": 23315, "east": 23316, "account": 23317, "Give": 23318, "\u0120transplant": 23319, "redients": 23320, "idable": 23321, "\u0120screenshots": 23322, "\u0120Gund": 23323, "\u0120FS": 23324, "\u0120travelers": 23325, "\u0120sensory": 23326, "\u0120Fiat": 23327, "\u0120Rockets": 23328, "\u0130\u012d": 23329, "_{": 23330, "Friend": 23331, "\u0120charming": 23332, "ALS": 23333, "\u0120enjoyment": 23334, "mph": 23335, "\u01205000": 23336, "\u0120REG": 23337, "\u00d9\u0128": 23338, "bia": 23339, "\u0120compilation": 23340, "rost": 23341, "\u0120VP": 23342, "\u0120Schne": 23343, "2019": 23344, "\u0120copying": 23345, "MORE": 23346, "\u0120Flore": 23347, "falls": 23348, "215": 23349, "total": 23350, "\u0120disciples": 23351, "double": 23352, "\u0120exceeding": 23353, "\u0120smashed": 23354, "\u0120conceptual": 23355, "\u0120Romania": 23356, "\u0120Brent": 23357, "\u0120ICE": 23358, "\u0120Tou": 23359, "\u0120grap": 23360, "\u0120nails": 23361, "189": 23362, "\u00e3\u0125\u013a": 23363, "\u0120procure": 23364, "eur": 23365, "\u0120confirming": 23366, "\u0120Cec": 23367, "awi": 23368, "\u0120Eden": 23369, "\u0120ng": 23370, "\u0120engineered": 23371, "atics": 23372, "\u0120hooked": 23373, "\u0120disgusting": 23374, "\u0120Murder": 23375, "\u00e3\u0124\u00bf": 23376, "Library": 23377, "\u0120168": 23378, "Almost": 23379, "hematic": 23380, "Menu": 23381, "\u0120Notre": 23382, "\u0120Jur": 23383, "\u0120kidnapped": 23384, "\u0120hacker": 23385, "\u0120Jade": 23386, "\u0120creepy": 23387, "\u0120drawings": 23388, "\u0120Sponsor": 23389, "\u0120cyclists": 23390, "\u0120Goblin": 23391, "\u0120optimized": 23392, "\u0120staged": 23393, "\u0120McD": 23394, "between": 23395, "Age": 23396, "eno": 23397, "Sex": 23398, "\u0120Wide": 23399, "nings": 23400, "avis": 23401, "\u0120incapable": 23402, "\u0120Kob": 23403, "\u0120rewarding": 23404, "\u0120Lone": 23405, "olescent": 23406, "\u0120contracted": 23407, "\u0120sticky": 23408, "Jose": 23409, "Ball": 23410, "fest": 23411, "\u0120Input": 23412, "\u0120Recently": 23413, "\u0120tomat": 23414, "square": 23415, "Application": 23416, "\u0120nitrogen": 23417, "\u0120duplicate": 23418, "\u0120Recon": 23419, "\u0120Dear": 23420, "London": 23421, "\u0120intra": 23422, "\u0120dock": 23423, "\u0120outreach": 23424, "\u0120Million": 23425, "\u0120mammals": 23426, "ampton": 23427, "VAL": 23428, "\u0120snaps": 23429, "\u0120dos": 23430, "\u0120Whole": 23431, "\u0120Ready": 23432, "Try": 23433, "\u0120Winnipeg": 23434, "earance": 23435, "\u0120incurred": 23436, "renched": 23437, "\u0120NSW": 23438, "ilot": 23439, "raine": 23440, "\u0120cube": 23441, "got": 23442, "\u0120runway": 23443, "etermined": 23444, "\u0120Hawks": 23445, "\u0120survivor": 23446, "\u0120Wish": 23447, "\u0120Din": 23448, "\u0120DEF": 23449, "\u0120Vault": 23450, "187": 23451, "\u0120mushrooms": 23452, "\u0120crisp": 23453, "bey": 23454, "\u0120Discovery": 23455, "\u0120developmental": 23456, "\u0120paradigm": 23457, "\u0120chaotic": 23458, "\u0120Tsu": 23459, "\u0120333": 23460, "bons": 23461, "\u0120bacterial": 23462, "\u0120commits": 23463, "\u0120cosmic": 23464, "\u0120mega": 23465, "ocative": 23466, "\u0120Paint": 23467, "ophobic": 23468, "\u0120vain": 23469, "\u0120carved": 23470, "\u0120Thief": 23471, "\u0120Gul": 23472, "owship": 23473, "\u0120cites": 23474, "\u0120Edinburgh": 23475, "\u0120diminished": 23476, "\u0120acknowledges": 23477, "\u0120Kills": 23478, "\u0120microw": 23479, "\u0120Hera": 23480, "\u0120seniors": 23481, "\u0120whereby": 23482, "Hop": 23483, "atron": 23484, "\u0120unavailable": 23485, "\u0120Nate": 23486, "\u0120480": 23487, "\u0120slated": 23488, "\u0120Rebecca": 23489, "\u0120Battery": 23490, "\u0120grammar": 23491, "\u0120headset": 23492, "\u0120cursor": 23493, "\u0120excluding": 23494, "anye": 23495, "aundering": 23496, "ebin": 23497, "\u0120feasible": 23498, "\u0120Publishing": 23499, "\u0120Labs": 23500, "\u0120Cliff": 23501, "\u0120Ferrari": 23502, "\u0120pac": 23503, "visible": 23504, "marked": 23505, "pell": 23506, "\u0120polite": 23507, "\u0120staggering": 23508, "\u0120Galactic": 23509, "\u0120superst": 23510, "\u0120paran": 23511, "\u0120Officers": 23512, "\u00e3\u0122\u0123": 23513, "\u0120specifics": 23514, "ulus": 23515, "239": 23516, "\u0120Paste": 23517, "AMP": 23518, "\u0120Panama": 23519, "\u0120Delete": 23520, "anguard": 23521, "restrial": 23522, "\u0120heroic": 23523, "\u0120Dy": 23524, "\u00d8\u00a7\u00d9\u0126": 23525, "\u0120incumbent": 23526, "\u0120crunch": 23527, "tro": 23528, "\u0120scoop": 23529, "\u0120blogger": 23530, "\u0120sellers": 23531, "uren": 23532, "\u0120medicines": 23533, "\u0120Caps": 23534, "\u0120Animation": 23535, "oxy": 23536, "\u0120outward": 23537, "\u0120inquiries": 23538, "229": 23539, "\u0120psychologist": 23540, "\u0120Sask": 23541, "evil": 23542, "\u0120contaminated": 23543, "\u00e3\u0124\u00a8": 23544, "herence": 23545, "\u0120branded": 23546, "\u0120Abdul": 23547, "zh": 23548, "\u0120paragraphs": 23549, "\u0120mins": 23550, "\u0120correlated": 23551, "erb": 23552, "\u0120impart": 23553, "\u0120milestone": 23554, "\u0120Solutions": 23555, "otle": 23556, "\u0120undercover": 23557, "\u0120marched": 23558, "\u0120Chargers": 23559, "fax": 23560, "\u0120Secrets": 23561, "\u0120ruth": 23562, "weather": 23563, "\u0120feminine": 23564, "\u0120sham": 23565, "\u0120prestigious": 23566, "iggins": 23567, "\u0120sung": 23568, "history": 23569, "ettle": 23570, "ggie": 23571, "\u0120outdated": 23572, "oland": 23573, "\u0120perceptions": 23574, "\u0120Session": 23575, "\u0120Dodgers": 23576, "uj": 23577, "\u0120END": 23578, "Doc": 23579, "\u0120deficiency": 23580, "Grand": 23581, "\u0120Joker": 23582, "\u0120retrospect": 23583, "\u0120diagnostic": 23584, "\u0120harmless": 23585, "\u0120rogue": 23586, "\u0120Aval": 23587, "Equ": 23588, "\u0120transc": 23589, "\u0120Robertson": 23590, "\u0120Depending": 23591, "\u0120Burns": 23592, "ivo": 23593, "\u0120hostility": 23594, "Features": 23595, "\u0135\u013a": 23596, "\u0120discomfort": 23597, "\u0120LCD": 23598, "specified": 23599, "\u0120Expect": 23600, "340": 23601, "\u0120imperative": 23602, "\u0120Regular": 23603, "Chinese": 23604, "\u0120statewide": 23605, "\u0120symm": 23606, "\u0120loops": 23607, "\u0120autumn": 23608, "Nick": 23609, "\u0120shaping": 23610, "\u0120quot": 23611, "\u0120cherry": 23612, "\u0120Crossref": 23613, "\u00e8\u00a6\u013c\u00e9\u0128\u0134": 23614, "Standard": 23615, "heed": 23616, "\u0120Dell": 23617, "\u0120Vietnamese": 23618, "\u0120ost": 23619, "\u0120Valkyrie": 23620, "OA": 23621, "Assad": 23622, "\u0120rebound": 23623, "\u0120Traffic": 23624, "places": 23625, "\u00e6\u013a": 23626, "\u0120Buc": 23627, "172": 23628, "\u0120shelters": 23629, "\u0120insisting": 23630, "\u0120Certainly": 23631, "\u0120Kenneth": 23632, "\u0120TCP": 23633, "\u0120penal": 23634, "\u0120Replay": 23635, "heard": 23636, "\u0120dialect": 23637, "iza": 23638, "\u0120FY": 23639, "itcher": 23640, "\u0120DL": 23641, "\u0120spiral": 23642, "\u0120quarterbacks": 23643, "\u0120hull": 23644, "\u0120google": 23645, "\u0120todd": 23646, "\u0120Sterling": 23647, "\u0120Plate": 23648, "\u0120spying": 23649, "mbol": 23650, "\u0120Realm": 23651, "\u0120Proced": 23652, "\u0120Crash": 23653, "\u0120terminate": 23654, "\u0120protesting": 23655, "Center": 23656, "guided": 23657, "\u0120uncover": 23658, "\u0120boycott": 23659, "\u0120realizes": 23660, "sound": 23661, "\u0120pretending": 23662, "\u0120Vas": 23663, "1980": 23664, "\u0120framed": 23665, "\u0120139": 23666, "\u0120descended": 23667, "\u0120rehabilitation": 23668, "\u0120borrowing": 23669, "\u0120Buch": 23670, "\u0120blur": 23671, "Ron": 23672, "\u0120Frozen": 23673, "enza": 23674, "Chief": 23675, "\u0120Poor": 23676, "\u0120translates": 23677, "MIN": 23678, "\u0120212": 23679, "JECT": 23680, "\u0120erupted": 23681, "\u0120successes": 23682, "SEC": 23683, "\u0120plague": 23684, "\u0120gems": 23685, "doms": 23686, "\u0120stretches": 23687, "\u0120Spy": 23688, "\u0120storytelling": 23689, "Credit": 23690, "\u0120Push": 23691, "\u0120traction": 23692, "\u0120ineffective": 23693, "\u0120Luna": 23694, "\u0120tapes": 23695, "\u0120analytics": 23696, "ercise": 23697, "\u0120programmes": 23698, "\u0120Carbon": 23699, "\u0120behold": 23700, "heavy": 23701, "\u0120Conservation": 23702, "\u0120FIR": 23703, "\u0120sack": 23704, "termin": 23705, "ricks": 23706, "\u0120housed": 23707, "\u0120unusually": 23708, "Ice": 23709, "\u0120executing": 23710, "\u0120Moroc": 23711, "eday": 23712, "\u0120editions": 23713, "\u0120smarter": 23714, "\u0120BA": 23715, "\u0120outlaw": 23716, "\u0120vanished": 23717, "iba": 23718, "ALSE": 23719, "\u0120Silva": 23720, "238": 23721, "Could": 23722, "\u0120philosopher": 23723, "\u0120evacuated": 23724, "Secret": 23725, "142": 23726, "\u0120visas": 23727, "\u00e3\u0124\u00ac": 23728, "\u0120Malt": 23729, "\u0120Clearly": 23730, "\u0120Niger": 23731, "\u0120Cairo": 23732, "\u0120Fist": 23733, "380": 23734, "\u0120XML": 23735, "auto": 23736, "itant": 23737, "\u0120reinforced": 23738, "Record": 23739, "\u0120Survivor": 23740, "GHz": 23741, "\u0120screws": 23742, "parents": 23743, "\u0120oceans": 23744, "mares": 23745, "\u0120brakes": 23746, "vasive": 23747, "\u0120hello": 23748, "\u0120SIM": 23749, "rimp": 23750, "\u0120ore": 23751, "\u0120Armour": 23752, "247": 23753, "\u0120terrific": 23754, "\u0120tones": 23755, "141": 23756, "\u0120Minutes": 23757, "Episode": 23758, "\u0120curves": 23759, "\u0120inflammatory": 23760, "\u0120batting": 23761, "\u0120Beautiful": 23762, "Lay": 23763, "\u0120unpop": 23764, "vable": 23765, "\u0120riots": 23766, "\u0120Tactics": 23767, "baugh": 23768, "\u0120Cock": 23769, "\u0120orgasm": 23770, "\u0120Sas": 23771, "\u0120constructor": 23772, "etz": 23773, "Gov": 23774, "\u0120antagon": 23775, "\u0120theat": 23776, "\u0120deeds": 23777, "hao": 23778, "cuts": 23779, "\u0120McCl": 23780, "\u0120um": 23781, "\u0120Scientists": 23782, "\u0120grassroots": 23783, "yssey": 23784, "\"]=>": 23785, "\u0120surfaced": 23786, "\u0120shades": 23787, "\u0120neighbours": 23788, "\u0120advertis": 23789, "oya": 23790, "\u0120merged": 23791, "Upon": 23792, "\u0120gad": 23793, "\u0120anticipate": 23794, "Anyway": 23795, "\u0120slogan": 23796, "\u0120disrespect": 23797, "Iran": 23798, "\u0120TB": 23799, "acted": 23800, "\u0120subpoen": 23801, "mediately": 23802, "OOOO": 23803, "\u0120waiver": 23804, "\u0120vulnerabilities": 23805, "ottesville": 23806, "\u0120Huffington": 23807, "Josh": 23808, "\u0120DH": 23809, "Monday": 23810, "\u0120Ellen": 23811, "Know": 23812, "xon": 23813, "items": 23814, "228": 23815, "\u0120fills": 23816, "\u0120Nike": 23817, "\u0120cumulative": 23818, "andals": 23819, "Ir": 23820, "\u0120\u00ec": 23821, "\u0120friction": 23822, "igator": 23823, "\u0120scans": 23824, "\u0120Vienna": 23825, "ldom": 23826, "\u0120performers": 23827, "Prim": 23828, "\u0120bidding": 23829, "Mur": 23830, "\u0120leaned": 23831, "\u0120Prix": 23832, "alks": 23833, "\u0120[\u00e2\u0122\u00a6]": 23834, "\u0120Twitch": 23835, "\u0120Developer": 23836, "\u0120Gir": 23837, "\u0120callback": 23838, "Abstract": 23839, "\u0120accustomed": 23840, "\u0120freedoms": 23841, "\u0120PG": 23842, "uracy": 23843, "\u0120lump": 23844, "isman": 23845, ",,,,": 23846, "1992": 23847, "\u0120RED": 23848, "\u0120worm": 23849, "Match": 23850, "\u0120Platinum": 23851, "IJ": 23852, "\u0120Owner": 23853, "Trivia": 23854, "compl": 23855, "\u0120newborn": 23856, "\u0120fantas": 23857, "Own": 23858, "\u01201959": 23859, "\u0120sympath": 23860, "\u0120ubiqu": 23861, "\u0120outputs": 23862, "\u0120allev": 23863, "\u0120prag": 23864, "Kevin": 23865, "\u0120favors": 23866, "\u0120burial": 23867, "\u0120nurt": 23868, "solete": 23869, "cache": 23870, "\u0120156": 23871, "\u0120unlocks": 23872, "techn": 23873, "Making": 23874, "\u0120conquer": 23875, "adic": 23876, "\u00e6\u0138": 23877, "\u0120elf": 23878, "\u0120electorate": 23879, "\u0120Kurds": 23880, "\u0120Stack": 23881, "\u0120Samurai": 23882, "\u0120\u00e2\u013a\u0127": 23883, "\u0120{}": 23884, "\u0120Said": 23885, "\u0120Fallout": 23886, "\u0120kindness": 23887, "\u0120Customs": 23888, "\u0120Boulevard": 23889, "\u0120helicopters": 23890, "otics": 23891, "\u0120Veget": 23892, "comment": 23893, "\u0120criticised": 23894, "\u0120polished": 23895, "\u0120Remix": 23896, "\u0120Cultural": 23897, "\u0120recons": 23898, "\u0120doi": 23899, "atem": 23900, "Screen": 23901, "\u0120barred": 23902, "Comments": 23903, "\u0120Generally": 23904, "\u0120slap": 23905, "720": 23906, "Vari": 23907, "pine": 23908, "\u0120empt": 23909, "\u0120hats": 23910, "\u0120Playing": 23911, "lab": 23912, "average": 23913, "forms": 23914, "\u0120Cotton": 23915, "\u0120cans": 23916, "\u0120DON": 23917, "\u0120Somalia": 23918, "Crypt": 23919, "\u0120Increases": 23920, "Ever": 23921, "modern": 23922, "\u0120surgeon": 23923, "3000": 23924, "\u0120randomized": 23925, "================================================================": 23926, "Bern": 23927, "impl": 23928, "\u0120COR": 23929, "\u0120proclaim": 23930, "thouse": 23931, "\u0120toes": 23932, "\u0120ample": 23933, "\u0120preserving": 23934, "\u0120disbel": 23935, "grand": 23936, "Besides": 23937, "\u0120silk": 23938, "\u0120Pattern": 23939, "hm": 23940, "\u0120enterprises": 23941, "\u0120affidavit": 23942, "\u0120Advisory": 23943, "\u0120advertised": 23944, "\u0120Religious": 23945, "sections": 23946, "psych": 23947, "\u0120Fields": 23948, "aways": 23949, "\u0120hashtag": 23950, "\u0120Nightmare": 23951, "\u0120vampire": 23952, "\u0120forensic": 23953, "rossover": 23954, "nar": 23955, "\u0120navy": 23956, "\u0120vacant": 23957, "\u0120Duel": 23958, "\u0120hallway": 23959, "\u0120facebook": 23960, "identally": 23961, "\u0120NRA": 23962, "\u0120matt": 23963, "\u0120hurricane": 23964, "\u0120Kirby": 23965, "\u0120Puzzle": 23966, "\u0120skirt": 23967, "oust": 23968, "dullah": 23969, "\u0120analogy": 23970, "inion": 23971, "\u0120tomatoes": 23972, "\u0120NV": 23973, "\u0120Peak": 23974, "\u0120Meyer": 23975, "\u0120appointments": 23976, "\u0120masc": 23977, "\u0120alley": 23978, "rehend": 23979, "\u0120charities": 23980, "\u0120undo": 23981, "\u0120destinations": 23982, "\u0120Testing": 23983, "\">\"": 24618, "cats": 24619, "*.": 24620, "\u0120gestures": 24621, "general": 24622, "League": 24623, "\u0120packets": 24624, "\u0120Inspector": 24625, "\u0120Berg": 24626, "\u0120fraudulent": 24627, "\u0120criticize": 24628, "Fun": 24629, "\u0120blaming": 24630, "ndra": 24631, "\u0120slash": 24632, "\u0120Eston": 24633, "\u0120proposing": 24634, "\u0120whales": 24635, "\u0120therapist": 24636, "\u0120subset": 24637, "\u0120leisure": 24638, "ELD": 24639, "\u0120CVE": 24640, "\u0120Activity": 24641, "\u0120culmin": 24642, "shop": 24643, "\u0120DAY": 24644, "ischer": 24645, "\u0120Admiral": 24646, "\u0120Attacks": 24647, "\u01201958": 24648, "\u0120memoir": 24649, "\u0120folded": 24650, "\u0120sexist": 24651, "\u0120153": 24652, "\u0120LI": 24653, "\u0120readings": 24654, "\u0120embarrassment": 24655, "\u0120Employment": 24656, "wart": 24657, "chin": 24658, "\u0120continuation": 24659, "lia": 24660, "Recently": 24661, "\u0120duel": 24662, "\u0120evacuation": 24663, "\u0120Kashmir": 24664, "\u0120disposition": 24665, "\u0120Rig": 24666, "\u0120bolts": 24667, "\u0120insurers": 24668, "467": 24669, "Mex": 24670, "\u0120retaliation": 24671, "\u0120misery": 24672, "\u0120unreasonable": 24673, "raining": 24674, "Imm": 24675, "\u0120PU": 24676, "emer": 24677, "\u0120genital": 24678, "\u00e3\u0124\u00b3": 24679, "\u0120Candy": 24680, "\u0120onions": 24681, "\u0120Patt": 24682, "liner": 24683, "\u0120conceded": 24684, "\u0120fa": 24685, "\u0120forc": 24686, "\u0120Hernandez": 24687, "\u0120Geoff": 24688, "debian": 24689, "\u0120Teams": 24690, "\u0120cries": 24691, "\u0120homeowners": 24692, "237": 24693, "ABC": 24694, "\u0120stitch": 24695, "\u0120statistic": 24696, "\u0120headers": 24697, "\u0120Biology": 24698, "\u0120motors": 24699, "\u0120GEN": 24700, "\u0120Lip": 24701, "\u0120hates": 24702, "\u0120heel": 24703, "Self": 24704, "ipl": 24705, "EDIT": 24706, "orting": 24707, "\u0120annot": 24708, "\u0120Speech": 24709, "oldemort": 24710, "\u0120Javascript": 24711, "\u0120LeBron": 24712, "\u0120footprint": 24713, "\u0120fn": 24714, "\u0120seizures": 24715, "nas": 24716, "hide": 24717, "\u01201954": 24718, "\u0120Bee": 24719, "\u0120Declaration": 24720, "\u0120Katie": 24721, "\u0120reservations": 24722, "NR": 24723, "female": 24724, "\u0120saturated": 24725, "\u0120biblical": 24726, "\u0120trolls": 24727, "Device": 24728, "photos": 24729, "\u0120drums": 24730, "\u00e3\u0125\u012b\u00e3\u0125\u00a9\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 24731, "Night": 24732, "fighter": 24733, "\u0120Hak": 24734, "riber": 24735, "\u0120cush": 24736, "\u0120disciplinary": 24737, "baum": 24738, "\u0120GH": 24739, "\u0120Schmidt": 24740, "ilibrium": 24741, "\u0120sixty": 24742, "\u0120Kushner": 24743, "rots": 24744, "\u0120pund": 24745, "\u0120Rac": 24746, "\u0120springs": 24747, "\u0120conve": 24748, "Business": 24749, "Fall": 24750, "\u0120qualifications": 24751, "\u0120verses": 24752, "\u0120narciss": 24753, "\u0120Koh": 24754, "\u0120Wow": 24755, "\u0120Charlottesville": 24756, "edo": 24757, "\u0120interrogation": 24758, "\u0120Wool": 24759, "365": 24760, "Brian": 24761, "\u0120\u00e2\u013e\u0135": 24762, "\u0120alleges": 24763, "onds": 24764, "idation": 24765, "\u0120Jackie": 24766, "yu": 24767, "\u0120lakes": 24768, "\u0120worthwhile": 24769, "\u0120crystals": 24770, "\u0120Juda": 24771, "\u0120comprehend": 24772, "\u0120flush": 24773, "\u0120absorption": 24774, "\u0120OC": 24775, "\u0120frightened": 24776, "\u0120Chocolate": 24777, "Martin": 24778, "\u0120buys": 24779, "\u0120bucks": 24780, "\u0120appell": 24781, "\u0120Championships": 24782, "\u0120listener": 24783, "\u0120Defensive": 24784, "\u0120cz": 24785, "uds": 24786, "\u0120Mate": 24787, "\u0120replay": 24788, "\u0120decorated": 24789, "\u0120sunk": 24790, "\u0120VIP": 24791, "\u0120Ank": 24792, "\u0120195": 24793, "aaaa": 24794, "Nobody": 24795, "\u0120Milk": 24796, "\u0120Gur": 24797, "\u0120Mk": 24798, "\u0120Sara": 24799, "\u0120seating": 24800, "\u0120Wid": 24801, "Track": 24802, "\u0120employs": 24803, "\u0120gigantic": 24804, "APP": 24805, "\u00e3\u0124\u00a7": 24806, "inventory": 24807, "\u0120towel": 24808, "atche": 24809, "lasting": 24810, "\u0120TL": 24811, "\u0120latency": 24812, "\u0120kne": 24813, "Ber": 24814, "meaning": 24815, "\u0120upheld": 24816, "\u0120playground": 24817, "\u0120mant": 24818, "Side": 24819, "\u0120stereo": 24820, "\u0120northwest": 24821, "\u0120exceptionally": 24822, "\u0120rays": 24823, "\u0120recurring": 24824, "Drive": 24825, "\u0120upright": 24826, "\u0120abduct": 24827, "\u0120Marathon": 24828, "\u0120goodbye": 24829, "\u0120alphabet": 24830, "hp": 24831, "\u0120courtroom": 24832, "rington": 24833, "othing": 24834, "Tag": 24835, "\u0120diplomats": 24836, "\u0120barbar": 24837, "\u0120Aqua": 24838, "183": 24839, "3333": 24840, "\u0120maturity": 24841, "\u0120instability": 24842, "\u0120Apache": 24843, "\u0120===": 24844, "\u0120fasting": 24845, "\u0120Grid": 24846, "ModLoader": 24847, "\u0120152": 24848, "Abs": 24849, "\u0120Operating": 24850, "etti": 24851, "\u0120acquaint": 24852, "Donnell": 24853, "\u0120Kem": 24854, "\u0120Forge": 24855, "\u0120armored": 24856, "Mil": 24857, "\u0120philosophers": 24858, "invest": 24859, "Players": 24860, "\u00e2\u012a": 24861, "\u0120myriad": 24862, "\u0120comrades": 24863, "Rot": 24864, "\u0120remembering": 24865, "\u0120corresponds": 24866, "\u0120programmers": 24867, "\u0120Lynn": 24868, "\u0120olig": 24869, "\u0120coherent": 24870, "ynchron": 24871, "\u0120Chemical": 24872, "\u0120jugg": 24873, "pair": 24874, "posts": 24875, "Eye": 24876, "\u0120Inner": 24877, "\u0120semester": 24878, "ottest": 24879, "\u0120Emirates": 24880, "ricanes": 24881, "orously": 24882, "mits": 24883, "\u0120Wis": 24884, "\u0120dodge": 24885, "location": 24886, "\u0120faded": 24887, "Amazon": 24888, "\u0120Proceed": 24889, "\u0120INFO": 24890, "journal": 24891, "\u0120Truck": 24892, "Ten": 24893, "\u0120217": 24894, "\u0120statutes": 24895, "mobile": 24896, "\u0120Types": 24897, "Recomm": 24898, "buster": 24899, "pex": 24900, "\u0120legends": 24901, "\u0120headache": 24902, "faced": 24903, "\u0120WiFi": 24904, "ifty": 24905, "\u0120HER": 24906, "\u0120circuits": 24907, "ERROR": 24908, "226": 24909, "olin": 24910, "\u0120cylinder": 24911, "ospace": 24912, "ikers": 24913, "Prem": 24914, "Quant": 24915, "\u0120conflicting": 24916, "\u0120slightest": 24917, "\u0120forged": 24918, "ionage": 24919, "Stephen": 24920, "\u0120Kub": 24921, "\u0120Opportun": 24922, "\u0120Heal": 24923, "\u0120blo": 24924, "\u0120rulers": 24925, "\u0120huh": 24926, "\u0120submarine": 24927, "fy": 24928, "asser": 24929, "\u0120allowance": 24930, "\u0120Kasich": 24931, "\u0120Tas": 24932, "\u0120Australians": 24933, "ForgeModLoader": 24934, "\u0120\u00e2\u0128\u0133": 24935, "\u0120Matrix": 24936, "amins": 24937, "\u01201200": 24938, "\u0120Acqu": 24939, "236": 24940, "Document": 24941, "\u0120Breaking": 24942, "193": 24943, "\u0120Subst": 24944, "\u0120Roller": 24945, "\u0120Properties": 24946, "\u0120NI": 24947, "tier": 24948, "\u0120crushing": 24949, "\u0120advocating": 24950, "Furthermore": 24951, "keepers": 24952, "\u0120sexism": 24953, "xd": 24954, "\u0120caller": 24955, "\u0120Sense": 24956, "chieve": 24957, "\u0120TF": 24958, "\u0120fueled": 24959, "\u0120reminiscent": 24960, "\u0120obsess": 24961, "urst": 24962, "\u0120uphold": 24963, "\u0120Fans": 24964, "hetics": 24965, "\u0120\u00e2\u0139": 24966, "\u0120Bath": 24967, "\u0120beverage": 24968, "\u0120oscill": 24969, "254": 24970, "\u0120poles": 24971, "\u0120gradual": 24972, "\u0120exting": 24973, "\u0120Suff": 24974, "\u0120Suddenly": 24975, "\u0120liking": 24976, "\u01201949": 24977, "unciation": 24978, "amination": 24979, "\u0120Omar": 24980, "\u0120LV": 24981, "\u0120Consequently": 24982, "\u0120synthes": 24983, "\u0120GIF": 24984, "\u0120pains": 24985, "\u0120interacting": 24986, "uously": 24987, "incre": 24988, "\u0120rumor": 24989, "\u0120Scientology": 24990, "197": 24991, "\u0120Zig": 24992, "\u0120spelling": 24993, "\u0120ASS": 24994, "\u0120extingu": 24995, "mson": 24996, "\u0120gh": 24997, "\u0120remarked": 24998, "\u0120Strategic": 24999, "\u0120MON": 25000, "\u00e5\u00a5": 25001, "gae": 25002, "\u0120WHAT": 25003, "Eric": 25004, "\u0120Campus": 25005, "\u0120methane": 25006, "\u0120imagin": 25007, "JUST": 25008, "\u0120Alm": 25009, "XT": 25010, "iq": 25011, "\u0120RSS": 25012, "\u0120wrongdoing": 25013, "atta": 25014, "\u0120bigot": 25015, "\u0120demonstrators": 25016, "\u0120Calvin": 25017, "\u0120Villa": 25018, "\u0120membrane": 25019, "\u0120Awesome": 25020, "\u0120benefic": 25021, "268": 25022, "\u0120magnificent": 25023, "\u0120Lots": 25024, "Greg": 25025, "\u0120Boris": 25026, "\u0120detainees": 25027, "\u0120Herman": 25028, "\u0120whispered": 25029, "\u0120awe": 25030, "Professor": 25031, "funding": 25032, "\u0120physiological": 25033, "\u0120Destruction": 25034, "\u0120limb": 25035, "\u0120manipulated": 25036, "\u0120bubbles": 25037, "\u0120pseud": 25038, "\u0120hydra": 25039, "\u0120Bristol": 25040, "\u0120stellar": 25041, "\u0120Expansion": 25042, "\u0120Kell": 25043, "\u0120Interestingly": 25044, "\u0120mans": 25045, "\u0120dragging": 25046, "\u0120ecological": 25047, "\u0120Fit": 25048, "\u0120gent": 25049, "\u0120benefited": 25050, "\u0120Haiti": 25051, "\u0120polyg": 25052, "\u00e3\u0125\u0130": 25053, "\u01202030": 25054, "\u0120prow": 25055, "\u0120reconstruction": 25056, "\u0120wast": 25057, "\u0120psychic": 25058, "\u0120Greeks": 25059, "Handler": 25060, "162": 25061, "\u0120Pulse": 25062, "\u0120solicit": 25063, "\u0120sys": 25064, "\u0120influx": 25065, "\u0120Gentle": 25066, "percent": 25067, "\u0120proliferation": 25068, "\u0120taxable": 25069, "\u0120disregard": 25070, "\u0120escaping": 25071, "\u0120ginger": 25072, "\u0120withstand": 25073, "\u0120devastated": 25074, "\u0120Dew": 25075, "series": 25076, "\u0120injected": 25077, "elaide": 25078, "\u0120turnover": 25079, "heat": 25080, "\u013b\u0124": 25081, "Happy": 25082, "\u0120Silent": 25083, "\u00e3\u0124\u0143": 25084, "ivism": 25085, "\u0120irrational": 25086, "AMA": 25087, "\u0120reef": 25088, "rub": 25089, "\u0120162": 25090, "\u0120bankers": 25091, "\u0120Ethics": 25092, "vv": 25093, "\u0120criticisms": 25094, "Kn": 25095, "186": 25096, "Movie": 25097, "\u0120Tories": 25098, "\u0120nood": 25099, "\u0120distortion": 25100, "False": 25101, "odore": 25102, "\u0120tasty": 25103, "Research": 25104, "\u0120UID": 25105, "-)": 25106, "\u0120divorced": 25107, "\u0120MU": 25108, "\u0120Hayes": 25109, "\u0120Isn": 25110, "iani": 25111, "\u0120HQ": 25112, "\u0120\"#": 25113, "ignant": 25114, "\u0120traumatic": 25115, "\u0120Ling": 25116, "Hun": 25117, "\u0120sabot": 25118, "online": 25119, "random": 25120, "\u0120renamed": 25121, "rared": 25122, "KA": 25123, "dead": 25124, "\u00c3\u00a9t": 25125, "\u0120Assistance": 25126, "\u0120seaf": 25127, "++++++++": 25128, "\u0120seldom": 25129, "\u0120Webb": 25130, "\u0120boolean": 25131, "ulet": 25132, "\u0120refrain": 25133, "\u0120DIY": 25134, "rule": 25135, "\u0120shutting": 25136, "\u0120utilizing": 25137, "loading": 25138, "\u0120Param": 25139, "coal": 25140, "ooter": 25141, "\u0120attracting": 25142, "\u0120Dol": 25143, "\u0120hers": 25144, "agnetic": 25145, "\u0120Reach": 25146, "imo": 25147, "\u0120discarded": 25148, "\u0120Pip": 25149, "015": 25150, "\u00c3\u00bcr": 25151, "\u0120mug": 25152, "Imagine": 25153, "COL": 25154, "\u0120cursed": 25155, "\u0120Shows": 25156, "\u0120Curtis": 25157, "\u0120Sachs": 25158, "speaking": 25159, "\u0120Vista": 25160, "\u0120Framework": 25161, "ongo": 25162, "\u0120subreddit": 25163, "\u0120crus": 25164, "\u0120Oval": 25165, "Row": 25166, "growing": 25167, "\u0120installment": 25168, "\u0120glac": 25169, "\u0120Advance": 25170, "ECK": 25171, "\u0120LGBTQ": 25172, "LEY": 25173, "\u0120acet": 25174, "\u0120successive": 25175, "\u0120Nicole": 25176, "\u01201957": 25177, "Quote": 25178, "\u0120circumstance": 25179, "ackets": 25180, "\u0120142": 25181, "ortium": 25182, "\u0120guessed": 25183, "\u0120Frame": 25184, "\u0120perpetrators": 25185, "\u0120Aviation": 25186, "\u0120Bench": 25187, "\u0120handc": 25188, "Ap": 25189, "\u01201956": 25190, "259": 25191, "rand": 25192, "NetMessage": 25193, "din": 25194, "urtles": 25195, "hig": 25196, "\u0120VIII": 25197, "ffiti": 25198, "\u0120Swords": 25199, "bial": 25200, "\u0120kidnapping": 25201, "device": 25202, "\u0120barn": 25203, "\u0120Eli": 25204, "aucas": 25205, "Send": 25206, "Constructed": 25207, "\u0120\u00c2\u00bd": 25208, "\u0120needles": 25209, "\u0120advertisements": 25210, "\u0120vou": 25211, "\u0120exhibited": 25212, "\u0120Fortress": 25213, "Ask": 25214, "Berry": 25215, "TYPE": 25216, "\u0120cancers": 25217, "umping": 25218, "\u0120Territory": 25219, "\u0120prud": 25220, "\u0120nas": 25221, "\u0120atheist": 25222, "\u0120balances": 25223, "\u00e3\u0123\u0141": 25224, "\u0120Shawn": 25225, "&&": 25226, "\u0120landsc": 25227, "\u0120RGB": 25228, "\u0120petty": 25229, "\u0120excellence": 25230, "\u0120translations": 25231, "\u0120parcel": 25232, "\u0120Chev": 25233, "East": 25234, "\u0120Output": 25235, "imi": 25236, "\u0120ambient": 25237, "\u0120Threat": 25238, "\u0120villains": 25239, "\u0120550": 25240, "ICA": 25241, "\u0120taller": 25242, "\u0120leaking": 25243, "cup": 25244, "\u0120polish": 25245, "\u0120infectious": 25246, "\u0120KC": 25247, "\u0120@@": 25248, "background": 25249, "\u0120bureaucracy": 25250, "\u0120Sai": 25251, "unless": 25252, "itious": 25253, "\u0120Skype": 25254, "Atl": 25255, "IDENT": 25256, "008": 25257, "\u0120hypocr": 25258, "\u0120pitchers": 25259, "\u0120guessing": 25260, "\u0120FINAL": 25261, "Between": 25262, "\u0120villagers": 25263, "\u0120252": 25264, "fashion": 25265, "\u0120Tunis": 25266, "Beh": 25267, "\u0120Exc": 25268, "\u0120MID": 25269, "288": 25270, "\u0120Haskell": 25271, "196": 25272, "\u0120NOR": 25273, "\u0120specs": 25274, "\u0120invari": 25275, "\u0120glut": 25276, "\u0120Cars": 25277, "\u0120impulse": 25278, "\u0120honors": 25279, "gel": 25280, "\u0120jurisdictions": 25281, "\u0120Bundle": 25282, "ulas": 25283, "California": 25284, "\u0120Increase": 25285, "\u0120pear": 25286, "\u0120singles": 25287, "\u0120cues": 25288, "\u0120underwent": 25289, "\u0120WS": 25290, "\u0120exaggerated": 25291, "\u0120dubious": 25292, "\u0120flashing": 25293, "LOG": 25294, ")].": 25295, "Journal": 25296, "tg": 25297, "Van": 25298, "\u0120Istanbul": 25299, "\u0120Insp": 25300, "\u0120Franken": 25301, "Draw": 25302, "\u0120sadness": 25303, "\u0120ironic": 25304, "\u0120Fry": 25305, "xc": 25306, "\u0120164": 25307, "isch": 25308, "Way": 25309, "\u0120Protestant": 25310, "horn": 25311, "\u0120unaff": 25312, "\u0120Viv": 25313, "illas": 25314, "\u0120Productions": 25315, "\u0120Hogan": 25316, "\u0120perimeter": 25317, "\u0120Sisters": 25318, "\u0120spontaneous": 25319, "\u0120downside": 25320, "\u0120descendants": 25321, "\u0120orn": 25322, "worm": 25323, "Japanese": 25324, "\u01201955": 25325, "\u0120151": 25326, "\u0120Doing": 25327, "elsen": 25328, "umbles": 25329, "\u0120radically": 25330, "\u0120Drum": 25331, "\u0120Bach": 25332, "\u0120liabilities": 25333, "\u0120OB": 25334, "\u0120Elementary": 25335, "\u0120meme": 25336, "ynes": 25337, "\u0120fingerprint": 25338, "\u0120Grab": 25339, "\u0120undertake": 25340, "Members": 25341, "\u0120Reader": 25342, "\u0120Sims": 25343, "god": 25344, "\u0120hypothetical": 25345, "scient": 25346, "\u0120AJ": 25347, "\u0120charism": 25348, "\u0120admissions": 25349, "\u0120Missile": 25350, "trade": 25351, "\u0120exercising": 25352, "\u0120Background": 25353, "Written": 25354, "\u0120vocals": 25355, "whether": 25356, "\u0120vi": 25357, "\u0120Winner": 25358, "\u0120litter": 25359, "\u0120Shooting": 25360, "STEM": 25361, "\u00e3\u0124\u00a1": 25362, "\u0120AFL": 25363, "\u0120variability": 25364, "\u0120eats": 25365, "\u0120DPS": 25366, "brow": 25367, "\u0120elephants": 25368, "\u0120strat": 25369, "\u0120\u00c5": 25370, "\u0120settlers": 25371, "Matthew": 25372, "\u0120inadvert": 25373, "HI": 25374, "\u0120IMF": 25375, "\u0120Goal": 25376, "\u0120nerves": 25377, "Johnson": 25378, "eye": 25379, "ablishment": 25380, "Thursday": 25381, "BILITY": 25382, "Had": 25383, "amoto": 25384, "hetamine": 25385, "eps": 25386, "\u0120mitochond": 25387, "\u0120compressed": 25388, "\u0120Trevor": 25389, "\u0120Animals": 25390, "Tool": 25391, "Lock": 25392, "\u0120tweak": 25393, "\u0120pinch": 25394, "\u0120cancellation": 25395, "Pot": 25396, "\u0120focal": 25397, "\u0120Astron": 25398, "173": 25399, "\u0120ASC": 25400, "\u0120OTHER": 25401, "umni": 25402, "\u0120demise": 25403, "dl": 25404, "\u00d9\u0127": 25405, "Semitism": 25406, "\u0120cracking": 25407, "\u0120collaborative": 25408, "\u0120explores": 25409, "sql": 25410, "\u0120herbs": 25411, "\u0120configurations": 25412, "mis": 25413, "\u0120Result": 25414, "acey": 25415, "\u0120Smoke": 25416, "\u0120sanct": 25417, "elia": 25418, "\u0120degener": 25419, "\u0120deepest": 25420, "\u0120screamed": 25421, "\u0120nap": 25422, "Software": 25423, "\u0120STAR": 25424, "EF": 25425, "\u0120Xin": 25426, "sponsored": 25427, "manship": 25428, "233": 25429, "\u0120primaries": 25430, "\u0120filtering": 25431, "\u0120assemble": 25432, "mil": 25433, "\u0120Myers": 25434, "bows": 25435, "\u0120punched": 25436, "Mic": 25437, "\u0120innovations": 25438, "\u0120func": 25439, "ando": 25440, "\u0120fracking": 25441, "\u0120Vul": 25442, "\u00d0\u00be\u00d0": 25443, "oshop": 25444, "\u0120Immun": 25445, "\u0120settling": 25446, "\u0120adolescents": 25447, "\u0120rebuilding": 25448, "\u0120transforming": 25449, "\u0120parole": 25450, "\u0120harbor": 25451, "\u0120booking": 25452, "otional": 25453, "ongevity": 25454, "\u0120Yo": 25455, "bug": 25456, "\u0120emerges": 25457, "\u0120Methods": 25458, "\u0120Chu": 25459, "Pres": 25460, "\u0120Dungeons": 25461, "\u0120trailing": 25462, "\u0120Rum": 25463, "\u0120Hugh": 25464, "\u00e5\u00a4\u00a9": 25465, "\u0120Era": 25466, "\u0120Battles": 25467, "Results": 25468, "\u0120Trading": 25469, "\u0120versa": 25470, "css": 25471, "axies": 25472, "heet": 25473, "\u0120greed": 25474, "1989": 25475, "\u0120gardens": 25476, "\u0120contingent": 25477, "Park": 25478, "\u0120Leafs": 25479, "hook": 25480, "robe": 25481, "\u0120diplomacy": 25482, "\u0120Fuel": 25483, "\u0120Invasion": 25484, "\u0120upgrading": 25485, "Male": 25486, "\u0120elic": 25487, "\u0120relentless": 25488, "\u0120Covenant": 25489, "apesh": 25490, "\u0120Trop": 25491, "Ty": 25492, "production": 25493, "arty": 25494, "\u0120punches": 25495, "ako": 25496, "cyclopedia": 25497, "\u0120Rabbit": 25498, "\u0120HDMI": 25499, "\u0120141": 25500, "\u0120foil": 25501, "ItemImage": 25502, "\u0120FG": 25503, "\u0120implementations": 25504, "\u0120Pom": 25505, "ixtures": 25506, "\u0120await": 25507, "\u0120330": 25508, "amus": 25509, "\u0120umbrella": 25510, "\u0120foresee": 25511, "separ": 25512, "\u0120circumcision": 25513, "\u0120peripheral": 25514, "Say": 25515, "\u0120Expert": 25516, "Inc": 25517, "\u0120withdrew": 25518, "\u0120Anders": 25519, "fried": 25520, "\u0120radioactive": 25521, "\u0120Opening": 25522, "\u0120boarding": 25523, "\u0120ND": 25524, "\u0120overthrow": 25525, "Activ": 25526, "WP": 25527, "\u0120Acts": 25528, "\u00d7\u013b": 25529, "\u0120motions": 25530, "vic": 25531, "\u0120Mighty": 25532, "\u0120Defender": 25533, "aer": 25534, "\u0120thankful": 25535, "\u0120Killing": 25536, "\u0120Bris": 25537, "moil": 25538, "\u0120predicting": 25539, "266": 25540, "choice": 25541, "\u0120killers": 25542, "\u0120incub": 25543, "\u0120Chest": 25544, "athering": 25545, "\u0120proclaimed": 25546, "flower": 25547, "ossom": 25548, "umbledore": 25549, "\u0120Cycling": 25550, "\u0120Occupy": 25551, "AGES": 25552, "Pen": 25553, "\u0120Yug": 25554, "\u0120packaged": 25555, "\u0120heightened": 25556, "cot": 25557, "stack": 25558, "Cond": 25559, "\u0120stamps": 25560, "mage": 25561, "\u0120persuaded": 25562, "\u0120ensl": 25563, "\u0120Cardinal": 25564, "\u0120solitary": 25565, "\u0120possessing": 25566, "\u0120Cork": 25567, "\u0120evid": 25568, "\u0120Tay": 25569, "\u0120blues": 25570, "\u0120extremism": 25571, "\u0120lunar": 25572, "\u0120clown": 25573, "Techn": 25574, "\u0120festivals": 25575, "\u0120PvP": 25576, "\u0120Lar": 25577, "\u0120consequently": 25578, "present": 25579, "\u0120someday": 25580, "\u00e7\u0130\u012d": 25581, "\u0120Meteor": 25582, "\u0120touring": 25583, "culture": 25584, "\u0120beaches": 25585, "Ship": 25586, "cause": 25587, "\u0120Flood": 25588, "\u00e3\u0125\u00af": 25589, "\u0120purity": 25590, "those": 25591, "\u0120emission": 25592, "bolt": 25593, "\u0120chord": 25594, "\u0120Scripture": 25595, "Lu": 25596, "\u0120${": 25597, "created": 25598, "Others": 25599, "258": 25600, "\u0120elemental": 25601, "\u0120annoyed": 25602, "\u0120AE": 25603, "dan": 25604, "\u0120Sag": 25605, "Researchers": 25606, "\u0120fairy": 25607, "\u00e2\u0122\u0135\u00e2\u0122\u0135": 25608, "============": 25609, "Smart": 25610, "GGGG": 25611, "\u0120skeletons": 25612, "\u0120pupils": 25613, "linked": 25614, "\u0120urgency": 25615, "enabled": 25616, "\u0120Fuck": 25617, "\u0120councill": 25618, "rab": 25619, "UAL": 25620, "TI": 25621, "\u0120lifes": 25622, "\u0120confessed": 25623, "Bug": 25624, "\u0120harmon": 25625, "\u0120CONFIG": 25626, "\u0120Neutral": 25627, "Double": 25628, "\u0120staple": 25629, "\u0120SHA": 25630, "British": 25631, "\u0120SNP": 25632, "ATOR": 25633, "oco": 25634, "\u0120swinging": 25635, "gex": 25636, "oleon": 25637, "plain": 25638, "\u0120Missing": 25639, "\u0120Trophy": 25640, "vari": 25641, "ranch": 25642, "\u0120301": 25643, "440": 25644, "0000000000000000": 25645, "\u0120restoring": 25646, "\u0120haul": 25647, "ucing": 25648, "nerg": 25649, "\u0120futures": 25650, "\u0120strategist": 25651, "question": 25652, "\u0120lateral": 25653, "\u0120Bard": 25654, "\u0120sor": 25655, "\u0120Rhodes": 25656, "\u0120Downtown": 25657, "?????-": 25658, "\u0120Lit": 25659, "\u0120Bened": 25660, "\u0120coil": 25661, "street": 25662, "\u0120Portal": 25663, "FILE": 25664, "\u0120Gru": 25665, "*,": 25666, "231": 25667, "neum": 25668, "\u0120sucked": 25669, "\u0120rapper": 25670, "\u0120tendencies": 25671, "\u0120Lauren": 25672, "cellaneous": 25673, "267": 25674, "\u0120browse": 25675, "\u0120overc": 25676, "header": 25677, "oise": 25678, "\u0120beet": 25679, "\u0120Gle": 25680, "Stay": 25681, "\u0120mum": 25682, "\u0120typed": 25683, "\u0120discounts": 25684, "Talk": 25685, "\u0120Og": 25686, "existing": 25687, "\u0120Sell": 25688, "uph": 25689, "CI": 25690, "\u0120Austrian": 25691, "\u0120Warm": 25692, "\u0120dismissal": 25693, "\u0120averages": 25694, "camera": 25695, "\u0120allegiance": 25696, "LAN": 25697, "=\"#": 25698, "\u0120commentators": 25699, "\u0120Setting": 25700, "\u0120Midwest": 25701, "\u0120pharmac": 25702, "\u0120EXP": 25703, "\u0120stainless": 25704, "Chicago": 25705, "\u0120tan": 25706, "244": 25707, "\u0120countryside": 25708, "\u0120Vac": 25709, "295": 25710, "\u0120pinned": 25711, "\u0120crises": 25712, "\u0120standardized": 25713, "Task": 25714, "\u0120Jail": 25715, "\u0120Docker": 25716, "colored": 25717, "forth": 25718, "\"},": 25719, "\u0120patrons": 25720, "\u0120spice": 25721, "\u0120mourn": 25722, "\u0120Mood": 25723, "\u0120laundry": 25724, "\u0120equip": 25725, "\u0120Mole": 25726, "yll": 25727, "\u0120THC": 25728, "nation": 25729, "\u0120Sherlock": 25730, "\u0120issu": 25731, "\u0120Kre": 25732, "\u0120Americas": 25733, "\u0120AAA": 25734, "\u0120systematically": 25735, "\u0120contra": 25736, "\u0120Sally": 25737, "\u0120rationale": 25738, "\u0120carriage": 25739, "\u0120peaks": 25740, "\u0120contradiction": 25741, "ensation": 25742, "\u0120Failure": 25743, "\u0120props": 25744, "\u0120namespace": 25745, "\u0120cove": 25746, "fields": 25747, "\u00e3\u0124\u012d": 25748, "\u0120wool": 25749, "\u0120Catch": 25750, "\u0120presumed": 25751, "\u0120Diana": 25752, "ragon": 25753, "igi": 25754, "\u0120hamm": 25755, "\u0120stunt": 25756, "\u0120GUI": 25757, "\u0120Observatory": 25758, "\u0120Shore": 25759, "\u0120smells": 25760, "annah": 25761, "\u0120cockpit": 25762, "\u0120Duterte": 25763, "850": 25764, "\u0120oppressed": 25765, "breaker": 25766, "\u0120Contribut": 25767, "\u0120Peru": 25768, "\u0120Monsanto": 25769, "\u0120Attempt": 25770, "\u0120commanding": 25771, "\u0120fridge": 25772, "\u0120Rin": 25773, "\u0120Chess": 25774, "uality": 25775, "\u0120ol": 25776, "Republican": 25777, "\u0120Glory": 25778, "\u0120WIN": 25779, ".......": 25780, "agent": 25781, "reading": 25782, "\u0120inh": 25783, "Jones": 25784, "\u0120clicks": 25785, "alan": 25786, "\u0120[];": 25787, "\u0120Majesty": 25788, "\u0120Ced": 25789, "opus": 25790, "atel": 25791, "\u00c3\u00aa": 25792, "ARC": 25793, "\u0120Ecuador": 25794, "\u00e3\u0125\u0142": 25795, "\u0120Kuro": 25796, "\u0120rituals": 25797, "\u0120captive": 25798, "\u0120ounce": 25799, "\u0120disagreement": 25800, "\u0120slog": 25801, "fuel": 25802, "Pet": 25803, "Mail": 25804, "\u0120exercised": 25805, "\u0120solic": 25806, "\u0120rainfall": 25807, "\u0120devotion": 25808, "\u0120Assessment": 25809, "\u0120robotic": 25810, "options": 25811, "\u0120RP": 25812, "\u0120Families": 25813, "\u0120Flames": 25814, "\u0120assignments": 25815, "007": 25816, "akedown": 25817, "\u0120vocabulary": 25818, "Reilly": 25819, "\u0120caval": 25820, "gars": 25821, "\u0120suppressed": 25822, "\u0120SET": 25823, "\u0120Johns": 25824, "\u0120warp": 25825, "broken": 25826, "\u0120statues": 25827, "\u0120advocated": 25828, "\u0120275": 25829, "\u0120peril": 25830, "omorph": 25831, "\u0120Femin": 25832, "perfect": 25833, "\u0120hatch": 25834, "Lib": 25835, "512": 25836, "\u0120lifelong": 25837, "313": 25838, "\u0120cheeks": 25839, "\u0120numbered": 25840, "\u0120Mug": 25841, "Body": 25842, "ravel": 25843, "Weight": 25844, "\u0120Jak": 25845, "\u0120Heath": 25846, "\u0120kissing": 25847, "\u0120JUST": 25848, "\u0120waving": 25849, "upload": 25850, "\u0120insider": 25851, "\u0120Progressive": 25852, "\u0120Filter": 25853, "tta": 25854, "\u0120Beam": 25855, "\u0120violently": 25856, "ipation": 25857, "\u0120skepticism": 25858, "\u01201918": 25859, "\u0120Annie": 25860, "\u0120SI": 25861, "\u0120genetics": 25862, "\u0120onboard": 25863, "atl": 25864, "\u0120Friedman": 25865, "\u0120Bri": 25866, "ceptive": 25867, "\u0120pirate": 25868, "\u0120Reporter": 25869, "278": 25870, "\u0120mythology": 25871, "\u0120eclipse": 25872, "\u0120skins": 25873, "\u0120glyph": 25874, "ingham": 25875, "Files": 25876, "Cour": 25877, "women": 25878, "\u0120regimes": 25879, "\u0120photographed": 25880, "Kat": 25881, "\u0120MAX": 25882, "Officials": 25883, "\u0120unexpectedly": 25884, "\u0120impressions": 25885, "Front": 25886, ";;;;;;;;": 25887, "\u0120supremacy": 25888, "\u0120sang": 25889, "\u0120aggravated": 25890, "\u0120abruptly": 25891, "\u0120Sector": 25892, "\u0120excuses": 25893, "\u0120costing": 25894, "idepress": 25895, "Stack": 25896, "\u0120RNA": 25897, "obil": 25898, "\u0120ghosts": 25899, "ldon": 25900, "atibility": 25901, "Topics": 25902, "\u0120reimburse": 25903, "\u0120HM": 25904, "\u0120Deg": 25905, "\u0120thief": 25906, "yet": 25907, "ogenesis": 25908, "leaning": 25909, "\u0120Kol": 25910, "\u0120Basketball": 25911, "\u0120fi": 25912, "\u0120Seeing": 25913, "\u0120recycling": 25914, "\u0120[-": 25915, "Congress": 25916, "\u0120lectures": 25917, "Psy": 25918, "\u0120nep": 25919, "\u0120maid": 25920, "\u0120oriented": 25921, "AX": 25922, "\u0120respectful": 25923, "rene": 25924, "flush": 25925, "\u0120Unloaded": 25926, "request": 25927, "grid": 25928, "\u0120Alternatively": 25929, "\u0120Hugo": 25930, "\u0120decree": 25931, "\u0120Buddhism": 25932, "andum": 25933, "Android": 25934, "\u0120Congo": 25935, "\u0120Joyce": 25936, "\u0120acknowledging": 25937, "hesive": 25938, "\u0120Tomorrow": 25939, "\u0120Hiro": 25940, "thren": 25941, "\u0120Maced": 25942, "\u0120hoax": 25943, "\u0120Increased": 25944, "\u0120Pradesh": 25945, "Wild": 25946, "______": 25947, "161": 25948, "\u0120aunt": 25949, "\u0120distributing": 25950, "\u0120Tucker": 25951, "\u0120SSL": 25952, "\u0120Wolves": 25953, "Building": 25954, "oult": 25955, "\u0120Luo": 25956, "\u0120Yas": 25957, "\u0120Spir": 25958, "\u0120Shape": 25959, "\u0120Cambod": 25960, "\u0120IPv": 25961, "\u0120ml": 25962, "\u0120extrad": 25963, "390": 25964, "\u0120Penny": 25965, "dream": 25966, "\u0120stationed": 25967, "optional": 25968, "eworthy": 25969, ".": 26700, "\u0120Workshop": 26701, "\u0120Retail": 26702, "\u0120Avatar": 26703, "625": 26704, "Na": 26705, "\u0120VC": 26706, "\u0120Secure": 26707, "MY": 26708, "1988": 26709, "ossip": 26710, "\u0120prostate": 26711, "\u0120unden": 26712, "\u0120gamer": 26713, "\u0120Contents": 26714, "\u0120Warhammer": 26715, "\u0120Sentinel": 26716, "310": 26717, "\u0120segregation": 26718, "\u0120Flex": 26719, "\u0120MAY": 26720, "\u0120drills": 26721, "\u0120Drugs": 26722, "Islamic": 26723, "\u0120spur": 26724, "\u0120cafe": 26725, "\u0120imaginary": 26726, "\u0120guiding": 26727, "\u0120swings": 26728, "\u0120Theme": 26729, "oby": 26730, "\u0120nud": 26731, "\u0120begging": 26732, "\u0120strongh": 26733, "\u0120rejecting": 26734, "\u0120pedestrians": 26735, "\u0120Prospect": 26736, "Rare": 26737, "sle": 26738, "\u0120concessions": 26739, "\u0120Constitutional": 26740, "\u0120beams": 26741, "\u0120fibers": 26742, "poon": 26743, "\u0120instincts": 26744, "property": 26745, "\u0120BIG": 26746, "Sanders": 26747, "imates": 26748, "\u0120coating": 26749, "\u0120corpses": 26750, "\u0120TRUE": 26751, "checked": 26752, "\u0120166": 26753, "Ash": 26754, "\u0120JS": 26755, "\u0120Fiction": 26756, "\u0120communal": 26757, "\u0120energetic": 26758, "oooooooo": 26759, "\u0120nowadays": 26760, "ILD": 26761, "ibo": 26762, "\u0120SUV": 26763, "Ren": 26764, "\u0120dwelling": 26765, "Silver": 26766, "\u0120tally": 26767, "\u0120Moving": 26768, "\u0120coward": 26769, "\u0120generals": 26770, "\u0120horns": 26771, "\u0120circulated": 26772, "\u0120robbed": 26773, "\u0120Unlimited": 26774, "\u0120harassed": 26775, "\u0120inhibit": 26776, "\u0120composer": 26777, "\u0120Spotify": 26778, "\u0120spreads": 26779, "364": 26780, "\u0120suicidal": 26781, "\u0120noises": 26782, "\u0120Stur": 26783, "\u0120saga": 26784, "\u0120Kag": 26785, "iso": 26786, "\u0120theoretically": 26787, "Money": 26788, "\u0120similarity": 26789, "\u0120sliced": 26790, "utils": 26791, "inges": 26792, "\"-": 26793, "\u0120anth": 26794, "\u0120imped": 26795, "Module": 26796, "Throughout": 26797, "\u0120menus": 26798, "committee": 26799, "andi": 26800, "obj": 26801, "inav": 26802, "fired": 26803, "\u0120Abdullah": 26804, "\u0120undead": 26805, "\u0120fonts": 26806, "Hold": 26807, "ENG": 26808, "\u0120sustainability": 26809, "\u0120flick": 26810, "\u0120razor": 26811, "\u0120Fest": 26812, "\u0120Characters": 26813, "\u0120wording": 26814, "\u0120populist": 26815, "\u0120criticizing": 26816, "\u0120muse": 26817, "vine": 26818, "\u0120cardboard": 26819, "\u0120kindly": 26820, "\u0120fringe": 26821, "\u0120Theft": 26822, "icultural": 26823, "\u0120governors": 26824, "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 26825, "\u0120163": 26826, "\u0120timeout": 26827, "\u0120Auth": 26828, "Children": 26829, "AU": 26830, "\u0120redemption": 26831, "\u0120Alger": 26832, "\u01201914": 26833, "\u0120waved": 26834, "\u0120astronauts": 26835, "ograms": 26836, "\u0120swamp": 26837, "\u0120Finnish": 26838, "\u0120candle": 26839, "\u0120tonnes": 26840, "utm": 26841, "\u0120ray": 26842, "\u0120spun": 26843, "\u0120fearful": 26844, "articles": 26845, "\u0120caus": 26846, "orically": 26847, "\u0120Requires": 26848, "\u0120Gol": 26849, "\u0120pope": 26850, "\u0120inaugural": 26851, "\u0120gle": 26852, "ADA": 26853, "\u0120ISIL": 26854, "\u0120Offensive": 26855, "\u0120watchdog": 26856, "\u0120balcon": 26857, "entity": 26858, "\u0120Hoo": 26859, "\u0120gallon": 26860, "ACC": 26861, "\u0120doubling": 26862, "\u0120implication": 26863, "\u0120Sight": 26864, "\u0120doctr": 26865, "-------": 26866, "\u0120\\\\": 26867, "\u0120malt": 26868, "Roll": 26869, "\u0120\u00e2\u012b\u00a5": 26870, "\u0120recap": 26871, "adding": 26872, "uces": 26873, "\u0120Bend": 26874, "figure": 26875, "\u0120turkey": 26876, "\u0120societal": 26877, "\u0120Tickets": 26878, "\u0120commercially": 26879, "\u0120spicy": 26880, "\u0120216": 26881, "\u0120Ramp": 26882, "\u0120superiority": 26883, "\u00c3\u00af": 26884, "\u0120Tracker": 26885, "Carl": 26886, "\u0120Coy": 26887, "\u0120Patriot": 26888, "\u0120consulted": 26889, "\u0120listings": 26890, "\u0120slew": 26891, "reenshot": 26892, "\u0120Gone": 26893, "\u0120[...]": 26894, "309": 26895, "\u0120hottest": 26896, "\u00d8\u00b1": 26897, "\u0120rocky": 26898, "\u0120Diaz": 26899, "\u0120massage": 26900, "\u0120paraly": 26901, "\u0120pony": 26902, "Az": 26903, "\u0120cartridge": 26904, "\u0120NZ": 26905, "\u0120snack": 26906, "\u0120Lamar": 26907, "plement": 26908, "\u0120Leslie": 26909, "\u0120mater": 26910, "\u0120snipp": 26911, "246": 26912, "\u0120jointly": 26913, "\u0120Brisbane": 26914, "\u0120iPod": 26915, "\u0120pumping": 26916, "\u0120goat": 26917, "\u0120Sharon": 26918, "ealing": 26919, "\u0120coron": 26920, "\u0120anomal": 26921, "rahim": 26922, "\u0120Connection": 26923, "\u0120sculpture": 26924, "\u0120scheduling": 26925, "\u0120Daddy": 26926, "athing": 26927, "\u0120eyebrows": 26928, "\u0120curved": 26929, "\u0120sentiments": 26930, "\u0120drafting": 26931, "Drop": 26932, "([": 26933, "\u0120nominal": 26934, "\u0120Leadership": 26935, "\u0120Grow": 26936, "\u0120176": 26937, "\u0120constructive": 26938, "ivation": 26939, "\u0120corrupted": 26940, "gerald": 26941, "\u0120Cros": 26942, "\u0120Chester": 26943, "\u0120Lap": 26944, "\u00e3\u0123\u00aa": 26945, "OTH": 26946, "DATA": 26947, "\u0120almond": 26948, "probably": 26949, "Imp": 26950, "\u0120feast": 26951, "\u0120Warcraft": 26952, "Flor": 26953, "\u0120checkpoint": 26954, "\u0120transcription": 26955, "\u0120204": 26956, "\u0120tweaks": 26957, "\u0120relieve": 26958, "Science": 26959, "\u0120performer": 26960, "Zone": 26961, "\u0120turmoil": 26962, "igated": 26963, "hibit": 26964, "\u0120Cafe": 26965, "themed": 26966, "\u0120fluor": 26967, "bench": 26968, "\u0120decom": 26969, "\u0120Unt": 26970, "\u0120Barrett": 26971, "\u0120Facts": 26972, "\u0120tasting": 26973, "\u0120PTSD": 26974, "\u0120Seal": 26975, "\u0120Judaism": 26976, "\u0120Dynamic": 26977, "\u0120Cors": 26978, "Ve": 26979, "\u0120Ming": 26980, "\u0120Transform": 26981, "von": 26982, "\u0120Defenders": 26983, "\u0120Tactical": 26984, "\u0120Von": 26985, "\u0120Univers": 26986, "\u0120distorted": 26987, "\u0120Breath": 26988, "?'\"": 26989, "\u0120agon": 26990, "\u0120Deadly": 26991, "\u0120lan": 26992, "\u0120Cycle": 26993, "orned": 26994, "\u0120reliably": 26995, "\u0120glor": 26996, "\u0120Monkey": 26997, "\u00e3\u0125\u00a1": 26998, "\u0120adren": 26999, "\u0120microwave": 27000, "\u0120Alban": 27001, "ircraft": 27002, "digit": 27003, "smart": 27004, "\u0120Dread": 27005, "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 27006, "{{": 27007, "\u0120Rochester": 27008, "\u0120simplified": 27009, "\u0120inflicted": 27010, "\u0120takeover": 27011, "\u0120yourselves": 27012, "aditional": 27013, "\u0120muscular": 27014, "KS": 27015, "\u0120ingen": 27016, "Tax": 27017, "\u0120Feature": 27018, "277": 27019, "\u0120cruc": 27020, "\u0120crate": 27021, "\u0120unidentified": 27022, "\u0120acclaimed": 27023, "\u0120Manga": 27024, "\u0120Frances": 27025, "\u0120Nepal": 27026, "\u0120Gerald": 27027, "\u0120Kuwait": 27028, "\u0120slain": 27029, "\u0120Heb": 27030, "\u0120Goku": 27031, "\u00e3\u0123\u00ae\u00e6": 27032, "286": 27033, "Mrs": 27034, "\u0120Cody": 27035, "\u0120Sanctuary": 27036, "016": 27037, "\u0120dismant": 27038, "\u0120dataset": 27039, "\u0120Hond": 27040, "buck": 27041, "\u0120Patterson": 27042, "\u0120palette": 27043, "\u0120GD": 27044, "icol": 27045, "\u0120Lodge": 27046, "\u0120planetary": 27047, "akin": 27048, "\u0120Registered": 27049, "abwe": 27050, "\u0120Petersburg": 27051, "\u0120hailed": 27052, "\u0120Piece": 27053, "Sche": 27054, "\u0120DOJ": 27055, "\u0120enumer": 27056, "181": 27057, "\u0120Observer": 27058, "\u0120Bold": 27059, "founded": 27060, "commerce": 27061, "\u0120exploits": 27062, "\u0120Finding": 27063, "URN": 27064, "\u0120Sne": 27065, "\u0120Acid": 27066, "ayette": 27067, "\u0120Values": 27068, "\u0120drastic": 27069, "\u0120architectural": 27070, "\u0120\".": 27071, "\u00d7\u0137": 27072, "umped": 27073, "\u0120wrapping": 27074, "\u0120widow": 27075, "\u0120Slayer": 27076, "lace": 27077, "once": 27078, "Germany": 27079, "avoid": 27080, "\u0120temples": 27081, "PAR": 27082, "\u00c3\u00b4": 27083, "\u0120Lucifer": 27084, "\u0120Flickr": 27085, "lov": 27086, "forces": 27087, "\u0120scouting": 27088, "\u0120louder": 27089, "tesy": 27090, "\u0120beforehand": 27091, "\u00c4\u0135": 27092, "\u0120Neon": 27093, "\u0120Wol": 27094, "\u0120Typically": 27095, "\u0120Politico": 27096, "-+-+": 27097, "\u0120builder": 27098, "\u0120derive": 27099, "Kill": 27100, "\u0120poker": 27101, "\u0120ambiguous": 27102, "\u0120lifts": 27103, "\u0120cyt": 27104, "\u0120ribs": 27105, "oodle": 27106, "\u0120Sounds": 27107, "hair": 27108, "\u0120Syndrome": 27109, "tf": 27110, "\u0120proportional": 27111, "uid": 27112, "\u0120pertaining": 27113, "\u0120Kindle": 27114, "\u0120Negro": 27115, "\u0120reiterated": 27116, "\u0120Tonight": 27117, "oths": 27118, "\u0120Cornell": 27119, "\u0120owing": 27120, "\u0120208": 27121, "elfare": 27122, "ocating": 27123, "\u0120Birds": 27124, "Subscribe": 27125, "\u0120essays": 27126, "\u0120burdens": 27127, "\u0120illustrations": 27128, "arious": 27129, "ERAL": 27130, "\u0120Calcul": 27131, "\u0120xen": 27132, "\u0120LinkedIn": 27133, "\u0120Jung": 27134, "\u0120redesign": 27135, "Connor": 27136, "296": 27137, "\u0120reversal": 27138, "\u0120Adelaide": 27139, "\u0120LL": 27140, "\u0120sinking": 27141, "\u0120gum": 27142, "USH": 27143, "capt": 27144, "\u0120Grimm": 27145, "\u0120footsteps": 27146, "\u0120CBD": 27147, "ispers": 27148, "\u0120prose": 27149, "Wednesday": 27150, "\u0120Movies": 27151, "edin": 27152, "\u0120overturned": 27153, "\u0120contentious": 27154, "USB": 27155, "~~~~~~~~~~~~~~~~": 27156, "\u0120Copper": 27157, "\u0120pointless": 27158, "NV": 27159, "values": 27160, "olphin": 27161, "dain": 27162, "\u0120deposited": 27163, "\u0120GW": 27164, "\u0120preceded": 27165, "\u0120Cla": 27166, "\u0120Golem": 27167, "\u0120Nim": 27168, "\u0120\u00ce\u00b2": 27169, "\u0120Engineers": 27170, "middle": 27171, "\u0120flatt": 27172, "operative": 27173, "\u0120councils": 27174, "imbabwe": 27175, "elin": 27176, "\u0120stressful": 27177, "\u0120LD": 27178, "\u0120resh": 27179, "lake": 27180, "\u0120wheelchair": 27181, "\u0120Alternative": 27182, "\u0120optimize": 27183, "operation": 27184, "\u0120peek": 27185, "\u0120oneself": 27186, "igil": 27187, "\u0120transitions": 27188, "opathy": 27189, "blank": 27190, "\u0120169": 27191, "171": 27192, "________________________________________________________________": 27193, "\u0120laundering": 27194, "Enc": 27195, "\u0120DEC": 27196, "\u0120workouts": 27197, "\u0120spikes": 27198, "\u0120dinosaurs": 27199, "\u0120discriminatory": 27200, "Pool": 27201, "Rather": 27202, "385": 27203, "RNA": 27204, "testers": 27205, "eto": 27206, "\u0120Identity": 27207, "\u0120vein": 27208, "\u0120Burton": 27209, "\u0120arcade": 27210, "420": 27211, "Ultimately": 27212, "\u0120Sadly": 27213, "\u00c3\u00b0": 27214, "pill": 27215, "\u0120cubic": 27216, "\u0120Spectrum": 27217, "these": 27218, "states": 27219, "\u0120unofficial": 27220, "hawks": 27221, "\u0120EVERY": 27222, "\u0120rainbow": 27223, "\u0120incarceration": 27224, "anding": 27225, "\u0120syll": 27226, "\u0120Everton": 27227, "\u0120179": 27228, "\u0120Serbia": 27229, "\u0120189": 27230, "meter": 27231, "\u0120Mickey": 27232, "\u0120antiqu": 27233, "\u0120factual": 27234, "neck": 27235, "\u0120Nare": 27236, "norm": 27237, "must": 27238, "\u0120highways": 27239, "\u0120glam": 27240, "\u0120dividing": 27241, "\u0120Squadron": 27242, "\u0120Martha": 27243, "\u0120births": 27244, "Cover": 27245, "////////////////": 27246, "\u0120Wong": 27247, "Phot": 27248, "\u0120ALS": 27249, "rio": 27250, "\u0120Nonetheless": 27251, "\u0120Lemon": 27252, "\u0120206": 27253, "\u0120EE": 27254, "\u0120derivative": 27255, "\u0120WWII": 27256, "vote": 27257, "\u0120therein": 27258, "\u0120separating": 27259, "446": 27260, "sync": 27261, "\u0120Streets": 27262, "\u0120ratt": 27263, "\u0120municipality": 27264, "\u0120Shortly": 27265, "\u0120monk": 27266, "),\"": 27267, "\u0120scrub": 27268, "\u0120operatives": 27269, "Neither": 27270, "Place": 27271, "\u0120Limit": 27272, "Female": 27273, "\u0120Actor": 27274, "Character": 27275, "\u0120constituted": 27276, "357": 27277, "\u0120protested": 27278, "\u0120Straw": 27279, "\u0120Height": 27280, "ilda": 27281, "\u0120Typh": 27282, "\u0120floods": 27283, "\u0120cosmetic": 27284, "WAY": 27285, "perture": 27286, "upon": 27287, "tons": 27288, "essing": 27289, "\u0120Pocket": 27290, "\u0120rooft": 27291, "\u0120Caucas": 27292, "\u0120antidepress": 27293, "\u0120incompatible": 27294, "ECD": 27295, "\u0120opera": 27296, "\u0120Contest": 27297, "\u0120generators": 27298, "lime": 27299, "Defense": 27300, "1987": 27301, "forum": 27302, "\u0120savage": 27303, "\u0120Hungarian": 27304, "nz": 27305, "\u0120metallic": 27306, "\u0120expelled": 27307, "\u0120residency": 27308, "\u0120dresses": 27309, "666": 27310, "\u0120Clement": 27311, "fires": 27312, "Category": 27313, "\u0120geek": 27314, "alis": 27315, "\u0120cemetery": 27316, "educated": 27317, "\u0120crawl": 27318, "\u0120Unable": 27319, "\u0120Tyson": 27320, "akis": 27321, "\u0120pardon": 27322, "\u0120Wra": 27323, "\u0120strengthened": 27324, "\u0120Fors": 27325, "335": 27326, "\u0120HC": 27327, "\u0120Mond": 27328, "\u0120visuals": 27329, "\u0120Beatles": 27330, "ettlement": 27331, "\u0120\u00ef": 27332, "gro": 27333, "\u0120bash": 27334, "\u0120poorest": 27335, "\u0120excel": 27336, "\u0120aspirations": 27337, "\u0120Municip": 27338, "ensible": 27339, "\u0120ceremonies": 27340, "\u0120intimidation": 27341, "\u0120CONTR": 27342, "beck": 27343, "\u0120Kap": 27344, "asu": 27345, "\u0120trademarks": 27346, "\u0120Sew": 27347, "\u0120Competition": 27348, "network": 27349, "\u0120Arri": 27350, "\u0120Tet": 27351, "Roaming": 27352, "WC": 27353, "Dat": 27354, "\u0120sob": 27355, "\u0120pairing": 27356, "\u0120overdose": 27357, "SAY": 27358, "aber": 27359, "\u0120revolt": 27360, "\u0120Fah": 27361, "acting": 27362, "eq": 27363, "estation": 27364, "Fight": 27365, "\u0120Marks": 27366, "273": 27367, "\u0120178": 27368, "Raw": 27369, "\u00e3\u0123\u012d": 27370, "349": 27371, "blocks": 27372, "\u0120verge": 27373, "estine": 27374, "\u0120Podesta": 27375, "\u0120invasive": 27376, "\u0120profoundly": 27377, "\u0120Ao": 27378, "each": 27379, "\u0120lest": 27380, "interpret": 27381, "\u0120shrinking": 27382, "\u0120errone": 27383, "\u0120chees": 27384, "lys": 27385, "\u0120Ivy": 27386, "\u0120Directory": 27387, "\u0120hinted": 27388, "VICE": 27389, "\u0120contacting": 27390, "\u0120Gent": 27391, "hei": 27392, "\u0120labeling": 27393, "\u0120mercury": 27394, "\u0120Lite": 27395, "\u0120expires": 27396, "\u0120destabil": 27397, "ritis": 27398, "cu": 27399, "\u0120feathers": 27400, "\u0120steer": 27401, "\u0120programmed": 27402, "\u0120Vader": 27403, "Going": 27404, "\u0120Elim": 27405, "\u0120yo": 27406, "\u0120Miche": 27407, "\u0120203": 27408, "\u0120sleeves": 27409, "\u0120bully": 27410, "\u0120Humans": 27411, "368": 27412, "\u0120compress": 27413, "\u0120Banner": 27414, "ARS": 27415, "\u0120awhile": 27416, "\u0120calib": 27417, "\u0120sponsorship": 27418, "\u0120Difficulty": 27419, "\u0120Papers": 27420, "\u0120identifier": 27421, "}.": 27422, "\u0120yog": 27423, "\u0120Shia": 27424, "\u0120cleanup": 27425, "\u0120vibe": 27426, "introdu": 27427, "imming": 27428, "Australia": 27429, "\u0120outlines": 27430, "\u0120Youtube": 27431, "train": 27432, "\u0120Makes": 27433, "\u0120deported": 27434, "\u0120centr": 27435, "\u0120Dug": 27436, "\u0120Boulder": 27437, "\u0120Buffy": 27438, "\u0120injunction": 27439, "\u0120Harley": 27440, "\u0120Groups": 27441, "\u0120Dumbledore": 27442, "\u0120Clara": 27443, "\u0120\"-": 27444, "\u0120sacrificed": 27445, "eph": 27446, "Shadow": 27447, "ibling": 27448, "\u0120freelance": 27449, "\u0120evidently": 27450, "phal": 27451, "\u0120retains": 27452, "Mir": 27453, "\u0120finite": 27454, "dar": 27455, "\u0120Cous": 27456, "\u0120repaired": 27457, "\u0120periodic": 27458, "\u0120championships": 27459, "\u0120asteroid": 27460, "blind": 27461, "\u0120expressly": 27462, "\u0120Astros": 27463, "\u0120scaled": 27464, "\u0120geographical": 27465, "\u0120Rapids": 27466, "Enjoy": 27467, "\u0120elastic": 27468, "\u0120Mohamed": 27469, "Market": 27470, "begin": 27471, "\u0120discovers": 27472, "\u0120telecommunications": 27473, "\u0120scanner": 27474, "\u0120enlarge": 27475, "\u0120sharks": 27476, "\u0120psychedel": 27477, "\u0120Rouge": 27478, "\u0120snapshot": 27479, "isine": 27480, "XP": 27481, "\u0120pesticides": 27482, "\u0120LSD": 27483, "\u0120Distribution": 27484, "really": 27485, "\u0120degradation": 27486, "\u0120disguise": 27487, "\u0120biom": 27488, "\u0120EXT": 27489, "\u0120equations": 27490, "\u0120hazards": 27491, "\u0120Compared": 27492, ")*": 27493, "\u0120virtues": 27494, "\u0120elders": 27495, "\u0120enhancing": 27496, "\u0120Across": 27497, "eros": 27498, "angling": 27499, "\u0120combust": 27500, "ucci": 27501, "\u0120concussion": 27502, "\u0120contraception": 27503, "\u0120Kang": 27504, "\u0120expresses": 27505, "\u0120aux": 27506, "\u0120Pione": 27507, "\u0120exhibits": 27508, "Debug": 27509, "OTAL": 27510, "\u0120Already": 27511, "\u0120Wheeler": 27512, "\u0120expands": 27513, "?:": 27514, "\u0120reconciliation": 27515, "\u0120pirates": 27516, "\u0120purse": 27517, "\u0120discourage": 27518, "\u0120spectacle": 27519, "Rank": 27520, "\u0120wraps": 27521, "\u0120Thought": 27522, "\u0120impending": 27523, "Opp": 27524, "\u0120Anglo": 27525, "\u0120EUR": 27526, "\u0120screwed": 27527, "retched": 27528, "\u0120encouragement": 27529, "models": 27530, "\u0120confuse": 27531, "mmm": 27532, "\u0120Vitamin": 27533, "\u00e2\u0138\u0133\u00e2\u0138\u0133": 27534, "Cru": 27535, "\u0120knights": 27536, "\u0120discard": 27537, "\u0120bishops": 27538, "\u0120Wear": 27539, "\u0120Garrett": 27540, "kan": 27541, "\u00e3\u0125\u0141": 27542, "\u0120masculine": 27543, "capital": 27544, "\u0120Aus": 27545, "\u0120fatally": 27546, "thanks": 27547, "\u0120AU": 27548, "\u0120Gut": 27549, "1200": 27550, "\u012000000000": 27551, "\u0120surrog": 27552, "\u0120BIOS": 27553, "raits": 27554, "\u0120Watts": 27555, "\u0120resurrection": 27556, "\u0120Electoral": 27557, "\u0120Tips": 27558, "4000": 27559, "\u0120nutrient": 27560, "\u0120depicting": 27561, "\u0120sprink": 27562, "\u0120muff": 27563, "\u0120LIM": 27564, "\u0120Sample": 27565, "psc": 27566, "ibi": 27567, "generated": 27568, "\u0120specimens": 27569, "\u0120dissatisf": 27570, "\u0120tailored": 27571, "\u0120holdings": 27572, "\u0120Monthly": 27573, "\u0120Eat": 27574, "poons": 27575, "\u0120nec": 27576, "\u0120Cage": 27577, "\u0120Lotus": 27578, "\u0120Lantern": 27579, "\u0120frontier": 27580, "\u0120pensions": 27581, "\u0120joked": 27582, "\u0120Hardy": 27583, "=-=-=-=-": 27584, "rade": 27585, "UID": 27586, "\u0120rails": 27587, "\u0120emit": 27588, "\u0120slate": 27589, "\u0120smug": 27590, "\u0120spit": 27591, "\u0120Calls": 27592, "\u0120Jacobs": 27593, "feat": 27594, "\u0120UE": 27595, "\u0120restruct": 27596, "\u0120regeneration": 27597, "\u0120energies": 27598, "\u0120Connor": 27599, "OHN": 27600, "\u0120Cheese": 27601, "\u0120ger": 27602, "\u0120resurrect": 27603, "management": 27604, "NW": 27605, "\u0120presently": 27606, "\u0120Bruins": 27607, "Member": 27608, "\u0120Mang": 27609, "idan": 27610, "\u0120boosting": 27611, "wyn": 27612, "+.": 27613, "requisite": 27614, "\u0120NYPD": 27615, "\u0120Megan": 27616, "\u0120Conditions": 27617, "\u0120pics": 27618, "nesium": 27619, "\u0120Rash": 27620, "\u0120174": 27621, "\u0120Ducks": 27622, "\u0120embro": 27623, "zu": 27624, "onian": 27625, "religious": 27626, "\u0120craz": 27627, "\u0120ACA": 27628, "\u0120Zucker": 27629, "EMA": 27630, "\u0120Pros": 27631, "Weapon": 27632, "\u0120Knox": 27633, "\u0120Arduino": 27634, "\u0120stove": 27635, "\u0120heavens": 27636, "\u0120Purchase": 27637, "\u0120herd": 27638, "\u0120fundraiser": 27639, "Digital": 27640, "5000": 27641, "\u0120proponents": 27642, "/\u00e2\u0122\u012d": 27643, "\u0120jelly": 27644, "\u0120Visa": 27645, "\u0120monks": 27646, "\u0120advancement": 27647, "\u0120Wer": 27648, "\u0120187": 27649, "eus": 27650, "ertility": 27651, "\u0120fetal": 27652, "\u01201936": 27653, "Lo": 27654, "\u0120outfits": 27655, "\u0120staircase": 27656, "bomb": 27657, "\u0120customized": 27658, "clair": 27659, "Tree": 27660, "\u0120mapped": 27661, "\u0120Considering": 27662, "\u0120Torres": 27663, "\u0120methyl": 27664, "\u0120approximate": 27665, "\u0120doom": 27666, "\u0120Hansen": 27667, "\u0120crossover": 27668, "\u0120standalone": 27669, "\u00e4\u00bc": 27670, "\u0120invites": 27671, "\u0120graveyard": 27672, "\u0120hp": 27673, "DonaldTrump": 27674, "\u0120escort": 27675, "Gar": 27676, "\u0120predecessors": 27677, "\u0120hay": 27678, "\u0120enzyme": 27679, "\u0120Straight": 27680, "visors": 27681, "Ing": 27682, "aneously": 27683, "\u0120Applied": 27684, "\u0120fec": 27685, "\u0120Durant": 27686, "\u0120outspoken": 27687, "orb": 27688, "\u0120zeal": 27689, "\u0120disgrace": 27690, "').": 27691, "\u0120Cheng": 27692, "289": 27693, "\u0120Rena": 27694, "\u0120Suicide": 27695, "294": 27696, "\u0120outraged": 27697, "\u0120Newman": 27698, "\u0120Nvidia": 27699, "\u0120Aber": 27700, "\u0120Bers": 27701, "\u0120recreation": 27702, "Window": 27703, "\u0120DP": 27704, "xe": 27705, "\u0120pedoph": 27706, "\u0120fallout": 27707, "amboo": 27708, "\u0120presentations": 27709, "\u0120Apps": 27710, "\u0120html": 27711, "345": 27712, "\u0120XXX": 27713, "\u0120rubbing": 27714, "\u0120Leather": 27715, "\u0120humidity": 27716, "seys": 27717, "established": 27718, "\u0120Units": 27719, "646": 27720, "\u0120respectable": 27721, "Auto": 27722, "\u0120thriving": 27723, "\u0120Innovation": 27724, "angs": 27725, "Extra": 27726, "regulation": 27727, "298": 27728, "pick": 27729, "Examples": 27730, "\u0120CJ": 27731, "Attack": 27732, "\u0120dracon": 27733, "LT": 27734, "\u0120sticker": 27735, "rers": 27736, "\u0120sunny": 27737, "Iss": 27738, "regulated": 27739, "dim": 27740, "\u0120Abstract": 27741, "\u0120husbands": 27742, "Office": 27743, "omination": 27744, "itars": 27745, "ANGE": 27746, "ascal": 27747, "\u0120Kris": 27748, "\u0120Infantry": 27749, "\u0120malf": 27750, "\u0120Athe": 27751, "\u0120Rally": 27752, "balanced": 27753, "........................": 27754, "OUP": 27755, "\u0120molecule": 27756, "metics": 27757, "\u0120Split": 27758, "\u0120Instructions": 27759, "\u0120Nights": 27760, "cards": 27761, "\u0120tug": 27762, "\u0120cone": 27763, "\u00e5\u0143": 27764, "\u0120tx": 27765, "\u0120Discussion": 27766, "\u0120catastrophe": 27767, "ppe": 27768, "gio": 27769, "\u0120communism": 27770, "\u0120halted": 27771, "\u0120Guant": 27772, "clean": 27773, "\u0120Sched": 27774, "\u0120Kanye": 27775, "\u0120wander": 27776, "\u0120Seriously": 27777, "\u0120188": 27778, "ennial": 27779, "follow": 27780, "productive": 27781, "\u0120Flow": 27782, "\u0120Sail": 27783, "\u0120craw": 27784, "\u0120simulations": 27785, "oru": 27786, "angles": 27787, "\u0120Nolan": 27788, "\u0120menstru": 27789, "470": 27790, "\u0120207": 27791, "aja": 27792, "\u0120casually": 27793, "boarding": 27794, "\u0120222": 27795, "ovy": 27796, "\u0120Numbers": 27797, "umat": 27798, "OE": 27799, "287": 27800, "\u0120Clemson": 27801, "\u0120certs": 27802, "\u0120slid": 27803, "\u0120Tribe": 27804, "\u0120toast": 27805, "\u0120fortunes": 27806, "\u0120fals": 27807, "\u0120Committees": 27808, "\u0120gp": 27809, "\u0120fiery": 27810, "\u0120Nets": 27811, "\u0120Anime": 27812, "Package": 27813, "\u0120Compare": 27814, "laughter": 27815, "infect": 27816, "\u0120atrocities": 27817, "\u0120justices": 27818, "\u0120insults": 27819, "\u0120Vernon": 27820, "\u0120shaken": 27821, "\u0120persona": 27822, "estamp": 27823, "367": 27824, "brain": 27825, "\u0120experimenting": 27826, "Ken": 27827, "\u0120Electronics": 27828, "\u0120161": 27829, "domain": 27830, "\u0120graphical": 27831, "bishop": 27832, "\u0120whopping": 27833, "\u0120Evangel": 27834, "\u0120advertisers": 27835, "\u0120Spear": 27836, "\u0120bids": 27837, "\u0120destroys": 27838, "utz": 27839, "\u0120undersc": 27840, "\u0120ADD": 27841, "\u0120ants": 27842, "\u0120Cum": 27843, "ipples": 27844, "\u0120Fill": 27845, "\u0120glanced": 27846, "\u0120indicted": 27847, "\u0120Eff": 27848, "\u0120miscon": 27849, "\u0120Desktop": 27850, "\u0120abide": 27851, "\u00e3\u0125\u0122": 27852, "\u0120Io": 27853, "\u0120Coul": 27854, "\u0120capsule": 27855, "\u0120Chrys": 27856, "MON": 27857, "\u0120undes": 27858, "\u0120IRA": 27859, "\u0120citation": 27860, "\u0120dictate": 27861, "\u0120Networks": 27862, "\u0120Conflict": 27863, "\u0120Stuff": 27864, "xa": 27865, "isec": 27866, "\u0120Chemistry": 27867, "\u0120quarterly": 27868, "Williams": 27869, "anan": 27870, "Opt": 27871, "\u0120Alexandria": 27872, "outheastern": 27873, "\u0120Springfield": 27874, "\u0120Blacks": 27875, "\u0120geography": 27876, "242": 27877, "\u0120utmost": 27878, "\u0120Exxon": 27879, "abouts": 27880, "EVA": 27881, "\u0120Enable": 27882, "\u0120Barr": 27883, "\u0120disagreed": 27884, "\u0120Cyprus": 27885, "\u0120dementia": 27886, "\u0120labs": 27887, "\u0120ubiquitous": 27888, "\u0120LOVE": 27889, "\u0120consolidated": 27890, "sr": 27891, "\u0120creamy": 27892, "\u0120Timber": 27893, "Regardless": 27894, "\u0120Certificate": 27895, "\u0120\"...": 27896, "ogenous": 27897, "Captain": 27898, "\u0120insulting": 27899, "\u0120Soros": 27900, "\u0120Instr": 27901, "\u0120Bulgaria": 27902, "better": 27903, "\u0120sucking": 27904, "\u0120Davidson": 27905, "atz": 27906, "\u0120collateral": 27907, "gif": 27908, "\u0120plagued": 27909, "\u0120Cancel": 27910, "\u0120Gardner": 27911, "RB": 27912, "\u0120sixteen": 27913, "Remove": 27914, "uristic": 27915, "cook": 27916, "Rod": 27917, "\u0120comprising": 27918, "fle": 27919, ")\u00e2\u0122\u0136": 27920, "\u0120Viking": 27921, "growth": 27922, "agonal": 27923, "\u0120srf": 27924, "afety": 27925, "mot": 27926, "Nearly": 27927, "stown": 27928, "\u0120Factor": 27929, "\u0120automobile": 27930, "\u0120procedural": 27931, "mask": 27932, "ampires": 27933, "\u0120disappears": 27934, "jab": 27935, "315": 27936, "\u01201951": 27937, "needed": 27938, "\u0120daring": 27939, "leader": 27940, "\u0120podium": 27941, "\u0120unhealthy": 27942, "\u0120mund": 27943, "\u0120pyramid": 27944, "ocre": 27945, "\u0120kissed": 27946, "\u0120dreamed": 27947, "\u0120Fantastic": 27948, "\u0120Gly": 27949, "\u00e5\u012c": 27950, "\u0120greatness": 27951, "\u0120spices": 27952, "\u0120metropolitan": 27953, "\u0120compuls": 27954, "iets": 27955, "1016": 27956, "\u0120Sham": 27957, "\u0120Pyr": 27958, "flies": 27959, "\u0120Midnight": 27960, "\u0120swallowed": 27961, "\u0120genres": 27962, "\u0120Lucky": 27963, "\u0120Rewards": 27964, "\u0120dispatch": 27965, "\u0120IPA": 27966, "\u0120Apply": 27967, "\u0120aven": 27968, "alities": 27969, "312": 27970, "things": 27971, "\u0120().": 27972, "\u0120mates": 27973, "\u0120Sz": 27974, "\u0120COP": 27975, "olate": 27976, "OFF": 27977, "\u0120recharge": 27978, "caps": 27979, "\u0120Yorker": 27980, "icone": 27981, "\u0120galaxies": 27982, "ileaks": 27983, "Dave": 27984, "\u0120Puzz": 27985, "\u0120Celtic": 27986, "\u0120AFC": 27987, "276": 27988, "\u0120Sons": 27989, "\u0120affirmative": 27990, "Hor": 27991, "\u0120tutorials": 27992, "\u0120CITY": 27993, "\u0120Rosa": 27994, "\u0120Extension": 27995, "Series": 27996, "\u0120fats": 27997, "\u0120rab": 27998, "lis": 27999, "\u0120unic": 28000, "\u0120eve": 28001, "\u0120Spin": 28002, "\u0120adulthood": 28003, "typ": 28004, "\u0120sectarian": 28005, "\u0120checkout": 28006, "\u0120Cycl": 28007, "Single": 28008, "\u0120martyr": 28009, "\u0120chilling": 28010, "888": 28011, "oufl": 28012, "\u0120];": 28013, "\u0120congestion": 28014, "mk": 28015, "\u0120Whereas": 28016, "\u01201938": 28017, "urrencies": 28018, "erion": 28019, "\u0120boast": 28020, "\u0120Patients": 28021, "\u0120chap": 28022, "\u0120BD": 28023, "realDonaldTrump": 28024, "\u0120examines": 28025, "hov": 28026, "\u0120startling": 28027, "\u0120Babylon": 28028, "wid": 28029, "omew": 28030, "brance": 28031, "\u0120Odyssey": 28032, "wig": 28033, "\u0120torch": 28034, "\u0120Vox": 28035, "\u0120Moz": 28036, "\u0120Troll": 28037, "\u0120Ans": 28038, "Similarly": 28039, "\u0120Ful": 28040, "006": 28041, "Unless": 28042, "\u0120Alone": 28043, "stead": 28044, "\u0120Publisher": 28045, "rights": 28046, "tu": 28047, "\u0120Doesn": 28048, "\u0120professionally": 28049, "\u0120clo": 28050, "icz": 28051, "\u0120steals": 28052, "\u0120\u00e1": 28053, "1986": 28054, "\u0120sturdy": 28055, "\u0120Johann": 28056, "\u0120medals": 28057, "\u0120filings": 28058, "\u0120Fraser": 28059, "done": 28060, "\u0120multinational": 28061, "\u0120feder": 28062, "\u0120worthless": 28063, "\u0120pest": 28064, "Yesterday": 28065, "ankind": 28066, "\u0120gays": 28067, "\u0120borne": 28068, "\u0120POS": 28069, "Picture": 28070, "\u0120percentages": 28071, "251": 28072, "rame": 28073, "\u0120potions": 28074, "AMD": 28075, "\u0120Lebanese": 28076, "\u0120rang": 28077, "\u0120LSU": 28078, "ongs": 28079, "\u0120peninsula": 28080, "\u0120Clause": 28081, "ALK": 28082, "oha": 28083, "\u0120MacBook": 28084, "\u0120unanimous": 28085, "\u0120lenders": 28086, "\u0120hangs": 28087, "\u0120franchises": 28088, "orers": 28089, "\u0120Updates": 28090, "\u0120isolate": 28091, "andro": 28092, "Soon": 28093, "\u0120disruptive": 28094, "\u0120Surve": 28095, "\u0120stitches": 28096, "\u0120Scorp": 28097, "\u0120Dominion": 28098, "\u0120supplying": 28099, "Arg": 28100, "\u0120turret": 28101, "\u0120Luk": 28102, "\u0120brackets": 28103, "*)": 28104, "\u0120Revolutionary": 28105, "\u0120Honest": 28106, "\u0120noticing": 28107, "\u0120Shannon": 28108, "\u0120afforded": 28109, "\u0120tha": 28110, "\u0120Janet": 28111, "!--": 28112, "\u0120Narendra": 28113, "\u0120Plot": 28114, "Hol": 28115, "sever": 28116, "eenth": 28117, "\u0120obstruction": 28118, "\u01201024": 28119, "staff": 28120, "jas": 28121, "orget": 28122, "scenes": 28123, "laughs": 28124, "\u0120Fargo": 28125, "crime": 28126, "\u0120orchestr": 28127, "\u0120delet": 28128, "iliary": 28129, "rieved": 28130, "\u0120militar": 28131, "\u0120Greene": 28132, "\u00e2\u0139\u0131": 28133, "\u00e3\u0123\u00a6": 28134, "\u0120Guards": 28135, "\u0120unleashed": 28136, "\u0120Weber": 28137, "\u0120adjustable": 28138, "\u0120caliber": 28139, "\u0120motivations": 28140, "\u0120\u00c3\u0142": 28141, "mAh": 28142, "\u0120Lanka": 28143, "handle": 28144, "\u0120pent": 28145, "\u0120Rav": 28146, "\u0120Angular": 28147, "\u0120Kau": 28148, "umbing": 28149, "\u0120philanthrop": 28150, "\u0120dehyd": 28151, "\u0120toxicity": 28152, "eer": 28153, "\u0120YORK": 28154, "witz": 28155, "\u00e5\u00bc": 28156, "\u0120IE": 28157, "community": 28158, "\u0120AH": 28159, "\u0120retali": 28160, "\u0120massively": 28161, "\u0120Daniels": 28162, "\u0120DEL": 28163, "\u0120carcin": 28164, "Url": 28165, "\u0120routing": 28166, "\u0120NPCs": 28167, "\u0120RAF": 28168, "ryce": 28169, "\u0120waived": 28170, "\u0120Guatem": 28171, "Everybody": 28172, "\u0120covenant": 28173, "\u0120173": 28174, "\u0120relaxing": 28175, "\u0120quart": 28176, "almost": 28177, "\u0120guarded": 28178, "\u0120Soldiers": 28179, "\u0120PLAY": 28180, "\u0120outgoing": 28181, "LAND": 28182, "\u0120rewrite": 28183, "\u0120MOV": 28184, "\u0120Imper": 28185, "\u0120Solution": 28186, "\u0120phenomenal": 28187, "\u0120longevity": 28188, "\u0120impat": 28189, "\u0120Nissan": 28190, "irie": 28191, "\u0120odor": 28192, "\u0120Zar": 28193, "oks": 28194, "\u0120militias": 28195, "\u0120SPEC": 28196, "\u0120tolerated": 28197, "arser": 28198, "\u0120Bradford": 28199, "+,": 28200, "\u0120surreal": 28201, "sf": 28202, "Canadian": 28203, "\u0120resemblance": 28204, "\u0120carbohydrate": 28205, "VIEW": 28206, "\u0120accessory": 28207, "meal": 28208, "largest": 28209, "iegel": 28210, "Someone": 28211, "\u0120toughest": 28212, "oso": 28213, "\u0120funnel": 28214, "\u0120condemnation": 28215, "luent": 28216, "\u0120wired": 28217, "\u0120Sunset": 28218, "Jesus": 28219, "\u0120PST": 28220, "\u0120Pages": 28221, "\u0120Tycoon": 28222, "\u0120PF": 28223, "\u0120selections": 28224, "\u0120\u00e0\u00a4": 28225, "partisan": 28226, "\u0120highs": 28227, "\u0120Rune": 28228, "\u0120crafts": 28229, "lead": 28230, "\u0120Parents": 28231, "\u0120reclaim": 28232, "eker": 28233, "\u0120Allied": 28234, "aeper": 28235, "\u0120looming": 28236, "\u0120beneficiaries": 28237, "\u0120Hull": 28238, "Students": 28239, "Jewish": 28240, "dj": 28241, "\u0120pact": 28242, "template": 28243, "\u0120Officials": 28244, "\u0120Baylor": 28245, "\u0120hemp": 28246, "\u0120youths": 28247, "\u0120Levels": 28248, "\u0120Xiao": 28249, "\u0120Ches": 28250, "\u0120endeavor": 28251, "\u0120Removed": 28252, "\u0120hippocamp": 28253, "Hell": 28254, "\u00e3\u0124\u012c": 28255, "805": 28256, "\u0120dinosaur": 28257, "\u0120Wrath": 28258, "\u0120Indonesian": 28259, "\u0120calculator": 28260, "\u0120Dictionary": 28261, "\u0120420": 28262, "\u0120MAG": 28263, "(_": 28264, "!,": 28265, "tarians": 28266, "\u0120restricting": 28267, "racuse": 28268, "\u0120weekday": 28269, "OUNT": 28270, "\u0120shrugged": 28271, "leground": 28272, "\u0120bald": 28273, "\u0120Doctors": 28274, "\u0120touted": 28275, "\u0120Maxwell": 28276, "\u0120214": 28277, "\u0120diplomat": 28278, "\u0120repression": 28279, "\u0120constituency": 28280, "vice": 28281, "ranked": 28282, "\u0120Napoleon": 28283, "gang": 28284, "\u0120Forever": 28285, "tun": 28286, "\u0120bulb": 28287, "\u0120PDT": 28288, "\u0120Cisco": 28289, "VEN": 28290, "\u0120resumed": 28291, "Steven": 28292, "\u0120Manitoba": 28293, "\u0120fabulous": 28294, "\u0120Agents": 28295, "1984": 28296, "\u0120amusing": 28297, "\u0120Mysteries": 28298, "\u0120orthodox": 28299, "floor": 28300, "\u0120questionnaire": 28301, "\u0120penetrate": 28302, "\u0120filmmakers": 28303, "\u0120Unc": 28304, "\u0120stamped": 28305, "\u0120thirteen": 28306, "\u0120outfield": 28307, "\u0120forwarded": 28308, "\u0120appra": 28309, "\u0120aided": 28310, "try": 28311, "\u0120unfocused": 28312, "\u0120Liz": 28313, "\u0120Wendy": 28314, "\u0120Scene": 28315, "Charg": 28316, "\u0120rejects": 28317, "\u0120leftist": 28318, "\u0120Providence": 28319, "\u0120Brid": 28320, "regn": 28321, "\u0120prophecy": 28322, "\u0120LIVE": 28323, "499": 28324, "\u0120forge": 28325, "\u0120FML": 28326, "\u0120intrinsic": 28327, "\u0120Frog": 28328, "\u0120wont": 28329, "\u0120Holt": 28330, "\u0120famed": 28331, "CLUS": 28332, "aepernick": 28333, "\u0120Hate": 28334, "\u0120Cay": 28335, "\u0120registering": 28336, "ortality": 28337, "ropy": 28338, "ocalyptic": 28339, "aan": 28340, "nav": 28341, "\u0120fascist": 28342, "IFIED": 28343, "\u0120implicated": 28344, "\u0120Resort": 28345, "\u0120Chandler": 28346, "\u0120Brick": 28347, "Pin": 28348, "ysc": 28349, "Usage": 28350, "\u0120Helm": 28351, "usra": 28352, "\u00e2\u013a\u0127\u00e2\u013a\u0127": 28353, "\u0120Abbas": 28354, "\u0120unanimously": 28355, "\u0120keeper": 28356, "\u0120addicted": 28357, "???": 28358, "\u0120helmets": 28359, "\u0120antioxid": 28360, "apsed": 28361, "808": 28362, "giene": 28363, "\u0120waits": 28364, "\u0120minion": 28365, "raved": 28366, "\u0120Porsche": 28367, "\u0120dreaming": 28368, "\u0120171": 28369, "\u0120Cain": 28370, "\u0120unfor": 28371, "asso": 28372, "\u0120Configuration": 28373, "kun": 28374, "hardt": 28375, "\u0120nested": 28376, "\u0120LDS": 28377, "LES": 28378, "\u0120tying": 28379, "enos": 28380, "\u0120cue": 28381, "\u0120Marqu": 28382, "skirts": 28383, "\u0120clicked": 28384, "\u0120expiration": 28385, "\u0120Accordingly": 28386, "\u0120WC": 28387, "\u0120blessings": 28388, "\u0120addictive": 28389, "\u0120Narr": 28390, "yx": 28391, "\u0120Jaguars": 28392, "\u0120rents": 28393, "\u0120Siber": 28394, "\u0120tipped": 28395, "ousse": 28396, "\u0120Fitzgerald": 28397, "\u0120hierarch": 28398, "outine": 28399, "\u0120wavelength": 28400, ">.": 28401, "chid": 28402, "\u0120Processing": 28403, "/+": 28404, "ranking": 28405, "Easy": 28406, "\u0120Construct": 28407, "\u0120tet": 28408, "insured": 28409, "HUD": 28410, "\u0120quoting": 28411, "\u0120communicated": 28412, "inx": 28413, "\u0120inmate": 28414, "\u0120erected": 28415, "\u0120Absolutely": 28416, "\u0120Surely": 28417, "\u0120unim": 28418, "\u0120Throne": 28419, "heid": 28420, "\u0120claws": 28421, "\u0120superstar": 28422, "\u0120Lenn": 28423, "\u0120Whis": 28424, "Uk": 28425, "abol": 28426, "\u0120sket": 28427, "\u0120Niet": 28428, "\u0120perks": 28429, "\u0120affinity": 28430, "\u0120openings": 28431, "phasis": 28432, "\u0120discriminate": 28433, "Tip": 28434, "vc": 28435, "\u0120grinding": 28436, "\u0120Jenny": 28437, "\u0120asthma": 28438, "holes": 28439, "\u0120Homer": 28440, "\u0120registers": 28441, "\u0120Glad": 28442, "\u0120creations": 28443, "\u0120lithium": 28444, "\u0120applause": 28445, "until": 28446, "Justice": 28447, "\u0120Turks": 28448, "\u0120scandals": 28449, "\u0120bake": 28450, "tank": 28451, "Mech": 28452, "\u0120Means": 28453, "\u0120Maid": 28454, "Republicans": 28455, "isal": 28456, "windows": 28457, "\u0120Santos": 28458, "\u0120vegetation": 28459, "338": 28460, "tri": 28461, "\u0120flux": 28462, "insert": 28463, "\u0120clarified": 28464, "\u0120mortg": 28465, "\u0120Chim": 28466, "\u0120Tort": 28467, "\u0120disclaim": 28468, "metal": 28469, "\u0120Aside": 28470, "\u0120induction": 28471, "\u0120infl": 28472, "\u0120atheists": 28473, "amph": 28474, "\u0120ether": 28475, "\u0120Vital": 28476, "\u0120Built": 28477, "Mind": 28478, "\u0120weaponry": 28479, "SET": 28480, "\u0120186": 28481, "admin": 28482, "gam": 28483, "contract": 28484, "afa": 28485, "\u0120derivatives": 28486, "\u0120snacks": 28487, "\u0120churn": 28488, "Econom": 28489, "\u0120capped": 28490, "\u0120Understanding": 28491, "\u0120Hers": 28492, "\u0120Iz": 28493, "\u0120duct": 28494, "IENT": 28495, "aughty": 28496, "\u0120\u00e2\u013e\u0136": 28497, "\u0120NP": 28498, "\u0120sailing": 28499, "Initialized": 28500, "\u0120ted": 28501, "\u0120reactors": 28502, "\u0120Lomb": 28503, "\u0120choke": 28504, "\u0120Worm": 28505, "\u0120admiration": 28506, "\u0120swung": 28507, "ensibly": 28508, "\u0120rash": 28509, "\u0120Goals": 28510, "\u0120Important": 28511, "Shot": 28512, "\u0120Ras": 28513, "\u0120trainers": 28514, "\u0120Bun": 28515, "Working": 28516, "\u0120harmed": 28517, "\u0120Pandora": 28518, "\u0120LTE": 28519, "\u0120mushroom": 28520, "\u0120CHAR": 28521, "\u0120Fee": 28522, "\u0120Moy": 28523, "Born": 28524, "oliberal": 28525, "\u0120Martial": 28526, "\u0120gentlemen": 28527, "\u0120lingering": 28528, "Official": 28529, "\u0120graffiti": 28530, "\u0120Names": 28531, "Der": 28532, "\u0120quint": 28533, "istrate": 28534, "azeera": 28535, "\u0120NOTICE": 28536, "\u0120Florence": 28537, "\u0120payable": 28538, "\u0120depicts": 28539, "\u0120Species": 28540, "Heart": 28541, "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 28542, "\u0120enclosed": 28543, "Increases": 28544, "Daily": 28545, "\u0120Lis": 28546, "\u0120enactment": 28547, "\u0120Bacon": 28548, "\u0120Steele": 28549, "demand": 28550, "\u0120183": 28551, "\u0120mouths": 28552, "\u0120stranded": 28553, "\u0120enhancement": 28554, "011": 28555, "\u0120Whats": 28556, "\u0120healed": 28557, "eny": 28558, "\u0120Rab": 28559, "\u0120340": 28560, "\u0120Labyrinth": 28561, "roach": 28562, "\u0120Yosh": 28563, "\u0120Clippers": 28564, "\u0120concerts": 28565, "Internet": 28566, "355": 28567, "\u0120stickers": 28568, "\u0120termed": 28569, "\u0120Axe": 28570, "\u0120grandparents": 28571, "France": 28572, "\u0120Clim": 28573, "\u0120Uh": 28574, "ulic": 28575, "\u0120thrill": 28576, "centric": 28577, "\u0120Overview": 28578, "\u0120Conduct": 28579, "\u0120substantive": 28580, "\u0120182": 28581, "mur": 28582, "\u0120stray": 28583, "\u0120Coff": 28584, "\u0120repetitive": 28585, "\u0120Forgotten": 28586, "\u0120qualification": 28587, "ewitness": 28588, "\u0120Zimbabwe": 28589, "\u0120simulated": 28590, "\u0120JD": 28591, "253": 28592, "\u0120Ware": 28593, "\u0120unsc": 28594, "Times": 28595, "\u0120summons": 28596, "\u0120disconnected": 28597, "\u0120184": 28598, "cius": 28599, "\u0120Gujar": 28600, "odka": 28601, "\u0120erase": 28602, "\u0120Tobacco": 28603, "elected": 28604, "\u0120uncont": 28605, "\u0120Shepard": 28606, "\u0120Lamp": 28607, "\u0120alerted": 28608, "\u0120operative": 28609, "arna": 28610, "uint": 28611, "\u0120negligence": 28612, "acements": 28613, "\u0120supra": 28614, "\u0120prevail": 28615, "\u0120Shark": 28616, "\u0120belts": 28617, "\u00e3\u0123\u00ab": 28618, "\u0120tighter": 28619, "Engineers": 28620, "\u0120inactive": 28621, "\u0120exponent": 28622, "\u0120Willie": 28623, "aples": 28624, "\u0120heir": 28625, "\u0120Hits": 28626, "iann": 28627, "\u0120Says": 28628, "\u0120currents": 28629, "\u0120Bengal": 28630, "\u0120arist": 28631, "Buffer": 28632, "\u0120breeze": 28633, "\u0120Wesley": 28634, "Cola": 28635, "\u0120pronoun": 28636, "\u0120deed": 28637, "\u0120Kling": 28638, "\u0120oft": 28639, "\u0120inflict": 28640, "\u0120punishing": 28641, "\u0120nm": 28642, "iku": 28643, "ODUCT": 28644, "014": 28645, "\u0120subsidy": 28646, "\u0120DEA": 28647, "\u0120Herbert": 28648, "\u0120Jal": 28649, "Bank": 28650, "\u0120deferred": 28651, "\u0120shipment": 28652, "Bott": 28653, "\u0120alle": 28654, "bearing": 28655, "HTML": 28656, "Offline": 28657, "\u0120213": 28658, "\u0120scrolling": 28659, "\u0120scanned": 28660, "\u0120Libyan": 28661, "\u0120TOP": 28662, "chrom": 28663, "dt": 28664, "column": 28665, "PsyNetMessage": 28666, "Zero": 28667, "\u0120torso": 28668, "050": 28669, "\u00e2\u0137\u0132": 28670, "\u0120imperson": 28671, "\u0120Schwartz": 28672, "udic": 28673, "\u0120pissed": 28674, "\u0120Sapp": 28675, "257": 28676, "\u0120ISPs": 28677, "ogl": 28678, "\u0120supervised": 28679, "\u0120adolescent": 28680, "\u0120attained": 28681, "\u0120Delivery": 28682, "\u0120Bunny": 28683, "\u01201937": 28684, "\u0120miniature": 28685, "\u0120os": 28686, "\u0120370": 28687, "608": 28688, "\u0120Mourinho": 28689, "\u0120innate": 28690, "\u0120tempo": 28691, "\u0120NM": 28692, "\u0120Fallen": 28693, "009": 28694, "\u0120provocative": 28695, "Streamer": 28696, "\u0120Benedict": 28697, "\u0120Bolshe": 28698, "\u0120turtle": 28699, "\u0120PCB": 28700, "\u0120Equal": 28701, "Director": 28702, "\u0120Rend": 28703, "\u0120fluids": 28704, "Authorities": 28705, "\u0120cousins": 28706, "requency": 28707, "\u0120Neighbor": 28708, "sets": 28709, "shared": 28710, "Charles": 28711, "password": 28712, "\u0120gears": 28713, "\u0120211": 28714, "\u0120Hardware": 28715, "rika": 28716, "\u0120upstream": 28717, "Hom": 28718, "\u0120disproportionately": 28719, "ivities": 28720, "\u0120undefined": 28721, "\u0120electrons": 28722, "\u0120commemor": 28723, "Eventually": 28724, "\u0120><": 28725, "\u0120irresponsible": 28726, "218": 28727, "\u0120Released": 28728, "\u0120OVER": 28729, "\u0120IGN": 28730, "\u0120Bread": 28731, "stellar": 28732, "\u0120Sage": 28733, "tted": 28734, "damage": 28735, "edition": 28736, "\u0120Prec": 28737, "\u0120lime": 28738, "\u0120confinement": 28739, "\u0120calorie": 28740, "weapon": 28741, "\u0120differing": 28742, "\u0120Sina": 28743, "mys": 28744, "amd": 28745, "\u0120intricate": 28746, "kk": 28747, "\u0120PAT": 28748, "\u00c3\u00a3o": 28749, "stones": 28750, "links": 28751, "\u0120ranch": 28752, "Semitic": 28753, "\u0120differentiate": 28754, "\u0120Singer": 28755, "occupied": 28756, "\u0120fortress": 28757, "cmd": 28758, "\u0120interception": 28759, "\u0120Ankara": 28760, "\u0120rept": 28761, "\u0120Solitaire": 28762, "\u0120remake": 28763, "pred": 28764, "\u0120dared": 28765, "autions": 28766, "\u0120BACK": 28767, "Running": 28768, "\u0120debugging": 28769, "\u0120graphs": 28770, "399": 28771, "\u0120Nigel": 28772, "\u0120bun": 28773, "\u0120pillow": 28774, "\u0120progressed": 28775, "fashioned": 28776, "\u0120obedience": 28777, "ERN": 28778, "\u0120rehears": 28779, "Cell": 28780, "tl": 28781, "Sher": 28782, "\u0120herald": 28783, "\u0120Payment": 28784, "\u0120Cory": 28785, "\u0120Dept": 28786, "\u0120repent": 28787, "\u0120Weak": 28788, "uckland": 28789, "\u0120pleasing": 28790, "\u0120shortages": 28791, "\u0120jurors": 28792, "\u0120Kab": 28793, "qqa": 28794, "Anti": 28795, "\u0120wow": 28796, "\u0120RCMP": 28797, "\u0120tsun": 28798, "\u0120Sic": 28799, "\u0120comprises": 28800, "\u0120spies": 28801, "\u0120precinct": 28802, "nu": 28803, "\u0120urges": 28804, "\u0120timed": 28805, "\u0120stripes": 28806, "\u0120Boots": 28807, "\u0120yen": 28808, "Advanced": 28809, "\u0120discrete": 28810, "\u0120Archangel": 28811, "employment": 28812, "Diff": 28813, "\u0120monuments": 28814, "\u0120209": 28815, "worker": 28816, "\u0120196": 28817, "\u0120Ig": 28818, "utterstock": 28819, "TPS": 28820, "Jac": 28821, "\u0120homelessness": 28822, "\u0120commentator": 28823, "\u0120racially": 28824, "fing": 28825, "seed": 28826, "Ele": 28827, "ellation": 28828, "\u0120ethanol": 28829, "\u0120parish": 28830, "\u0120Dong": 28831, "\u0120Awakening": 28832, "\u0120deviation": 28833, "\u0120Bearing": 28834, "\u0120Tsuk": 28835, "\u0120recess": 28836, "\u0120lymph": 28837, "\u0120Cannabis": 28838, "\u00e5\u013e": 28839, "\u0120NEWS": 28840, "\u0120dra": 28841, "\u0120Stefan": 28842, "\u0120Wrong": 28843, "\u0120SAM": 28844, "\u0120loosely": 28845, "\u0120interpreter": 28846, "\u0120Plain": 28847, "Government": 28848, "\u0120bigotry": 28849, "\u0120grenades": 28850, "avez": 28851, "pictured": 28852, "\u0120mandated": 28853, "\u0120Monk": 28854, "\u0120Pedro": 28855, "\u0120lava": 28856, "274": 28857, "\u0120cynical": 28858, "\u0120Scrolls": 28859, "locks": 28860, "Mp": 28861, "\u0120congregation": 28862, "ornings": 28863, "phil": 28864, "\u0120Ibid": 28865, "\u0120ferv": 28866, "\u0120disappearing": 28867, "\u0120arrogant": 28868, "syn": 28869, "\u0120Maver": 28870, "\u0120Suit": 28871, "241": 28872, "\u0120abbre": 28873, "ackers": 28874, "Pa": 28875, "\u0120Yel": 28876, "Whenever": 28877, "\u0120235": 28878, "\u0120Vine": 28879, "\u0120Anat": 28880, "\u0120extinct": 28881, "LET": 28882, "\u0120executable": 28883, "VERS": 28884, "oxide": 28885, "DNA": 28886, "\u0120Prel": 28887, "\u0120resentment": 28888, "\u0120comprise": 28889, "\u0120Aviv": 28890, "\u0120interceptions": 28891, "\u0120prolific": 28892, "INA": 28893, "\u0120Erin": 28894, "thought": 28895, "219": 28896, "\u0120Psychiatry": 28897, "unky": 28898, "chemist": 28899, "Ho": 28900, "\u0120McCoy": 28901, "\u0120bricks": 28902, "Los": 28903, "rily": 28904, "\u0120USSR": 28905, "\u0120rud": 28906, "\u0120laud": 28907, "\u0120Wise": 28908, "\u0120Emerald": 28909, "\u0120revived": 28910, "\u0120damned": 28911, "\u0120Repair": 28912, "idem": 28913, "ctica": 28914, "\u0120patriarch": 28915, "\u0120Nurs": 28916, "meg": 28917, "\u0120cheapest": 28918, "reements": 28919, "empty": 28920, "\u0120Celebr": 28921, "\u0120deprivation": 28922, "chanted": 28923, "\u0120Thumbnails": 28924, "Energy": 28925, "\u0120Ethan": 28926, "\u0120Qing": 28927, "\u0120opposes": 28928, "WIND": 28929, "vik": 28930, "\u0120Mau": 28931, "\u0120SUB": 28932, "667": 28933, "GRE": 28934, "\u0120Volunte": 28935, "nton": 28936, "Cook": 28937, "\u00e5\u0132": 28938, "esque": 28939, "\u0120plummet": 28940, "\u0120suing": 28941, "\u0120pronounce": 28942, "\u0120resisting": 28943, "\u0120Fishing": 28944, "\u0120Trials": 28945, "\u0120yell": 28946, "\u0120310": 28947, "\u0120induct": 28948, "\u0120personalized": 28949, "often": 28950, "Reb": 28951, "EMBER": 28952, "\u0120viewpoint": 28953, "\u0120existential": 28954, "())": 28955, "remove": 28956, "MENTS": 28957, "lasses": 28958, "\u0120evapor": 28959, "\u0120aisle": 28960, "meta": 28961, "\u0120reflective": 28962, "\u0120entitlement": 28963, "\u0120devised": 28964, "music": 28965, "ascade": 28966, "\u0120winding": 28967, "offset": 28968, "\u0120accessibility": 28969, "kered": 28970, "Better": 28971, "\u0120Johnston": 28972, "thinking": 28973, "Snow": 28974, "\u0120Croatia": 28975, "\u0120Atomic": 28976, "271": 28977, "348": 28978, "\u0120textbook": 28979, "\u0120Sixth": 28980, "\u0120\u00d8\u00a7\u00d9\u0126": 28981, "\u0120slider": 28982, "\u0120Burger": 28983, "bol": 28984, "Sync": 28985, "\u0120grandchildren": 28986, "\u0120cerv": 28987, "+)": 28988, "\u0120eternity": 28989, "\u0120tweeting": 28990, "\u0120speculative": 28991, "\u0120pivotal": 28992, "\u0120WP": 28993, "\u0120TER": 28994, "ynamic": 28995, "\u0120upl": 28996, "\u0120Cats": 28997, "perhaps": 28998, "\u0120classmates": 28999, "\u0120blatant": 29000, "'-": 29001, "\u0120lakh": 29002, "antine": 29003, "\u0120Borg": 29004, "iom": 29005, "/(": 29006, "\u0120Athletic": 29007, "\u0120sar": 29008, "OTA": 29009, "\u0120Hoffman": 29010, "Nevertheless": 29011, "\u0120adorable": 29012, "\u0120spawned": 29013, "Associated": 29014, "\u0120Domestic": 29015, "\u0120implant": 29016, "\u0120Luxem": 29017, "\u0120Kens": 29018, "\u0120pumps": 29019, "\u0120SAT": 29020, "Attributes": 29021, "509": 29022, "avour": 29023, "\u0120centralized": 29024, "\u0120TN": 29025, "\u0120freshly": 29026, "\u0120Achieve": 29027, "\u0120outsiders": 29028, "herty": 29029, "\u0120Ree": 29030, "\u0120Towers": 29031, "\u0120Dart": 29032, "akable": 29033, "\u0120mp": 29034, "\u0120Heavenly": 29035, "\u0120ripe": 29036, "\u0120Caroline": 29037, "ryan": 29038, "\u0120classics": 29039, "\u0120retiring": 29040, "\u0120228": 29041, "\u0120ah": 29042, "\u0120dealings": 29043, "\u0120punching": 29044, "\u0120Chapman": 29045, "Options": 29046, "maxwell": 29047, "volume": 29048, "\u0120stal": 29049, "\u0120exported": 29050, "\u0120Quite": 29051, "\u0120numerical": 29052, "Burn": 29053, "Fact": 29054, "\u0120Keystone": 29055, "\u0120trending": 29056, "\u0120altering": 29057, "\u0120Africans": 29058, "478": 29059, "\u0120MN": 29060, "\u0120Knock": 29061, "\u0120temptation": 29062, "\u0120prestige": 29063, "Overview": 29064, "\u0120Traditional": 29065, "\u0120Bahrain": 29066, "Private": 29067, "\u0120HOU": 29068, "\u0120barr": 29069, "\u0120Tat": 29070, "Cube": 29071, "USD": 29072, "\u0120Grande": 29073, "\u0120Gat": 29074, "\u0120Flo": 29075, "\u0120resides": 29076, "\u0120indec": 29077, "volent": 29078, "\u0120perpetual": 29079, "ubes": 29080, "\u0120worldview": 29081, "\u0120Quantum": 29082, "\u0120filtered": 29083, "\u0120ensu": 29084, "orgetown": 29085, "ERSON": 29086, "\u0120Mild": 29087, "379": 29088, "OTT": 29089, "\u00c3\u00a5": 29090, "\u0120vitamins": 29091, "\u0120ribbon": 29092, "\u0120sincerely": 29093, "\u0120Hin": 29094, "\u0120eighteen": 29095, "\u0120contradictory": 29096, "\u0120glaring": 29097, "\u0120expectancy": 29098, "\u0120conspir": 29099, "\u0120monstrous": 29100, "\u0120380": 29101, "reci": 29102, "\u0120handic": 29103, "\u0120pumped": 29104, "\u0120indicative": 29105, "\u0120rapp": 29106, "\u0120avail": 29107, "\u0120LEGO": 29108, "\u0120Marijuana": 29109, "1985": 29110, "erton": 29111, "\u0120twentieth": 29112, "################################": 29113, "\u0120Swamp": 29114, "\u0120valuation": 29115, "\u0120affiliates": 29116, "adjusted": 29117, "\u0120Facility": 29118, "262": 29119, "\u0120enzymes": 29120, "itudinal": 29121, "\u0120imprint": 29122, "Site": 29123, "\u0120installer": 29124, "\u0120TRA": 29125, "mology": 29126, "linear": 29127, "\u0120Collective": 29128, "igating": 29129, "\u0120Token": 29130, "\u0120speculated": 29131, "KN": 29132, "\u0120Cly": 29133, "ority": 29134, "\u0120defer": 29135, "\u0120inspectors": 29136, "approved": 29137, "RM": 29138, "\u0120Suns": 29139, "\u0120informing": 29140, "\u0120Syracuse": 29141, "ibli": 29142, "765": 29143, "\u0120glove": 29144, "\u0120authorize": 29145, "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 29146, "\u0120Cruise": 29147, "\u0120contracting": 29148, "shell": 29149, "IFE": 29150, "\u0120Jewel": 29151, "pract": 29152, "\u0120Photoshop": 29153, "\u0120Knowing": 29154, "harm": 29155, "\u0120attractions": 29156, "adan": 29157, "etus": 29158, "018": 29159, "wagen": 29160, "Alt": 29161, "\u0120multiply": 29162, "\u0120equilibrium": 29163, ":{": 29164, "\u0120Fighters": 29165, "\u0120Edgar": 29166, "\u0120fourteen": 29167, "Govern": 29168, "\u0120misuse": 29169, "\u0120abusing": 29170, "\u0120ancestry": 29171, "ramer": 29172, "644": 29173, "\u0120worms": 29174, "\u0120thicker": 29175, "\u0120Combine": 29176, "\u0120peasants": 29177, "\u0120vind": 29178, "\u0120conquest": 29179, "\u0120mocked": 29180, "\u0120cinnamon": 29181, "\u0120Cald": 29182, "\u0120Gallup": 29183, "\u0120avoidance": 29184, "\u0120incarnation": 29185, "\u0120Strat": 29186, "\u0120tasted": 29187, "enta": 29188, "\u0120Neal": 29189, "pared": 29190, "\u0120terminology": 29191, "jection": 29192, "Scientists": 29193, "\u0120INS": 29194, "\u0120Dee": 29195, "\u0120directories": 29196, "Road": 29197, "\u0120Shap": 29198, "bright": 29199, "\u0120Directors": 29200, "\u0120Column": 29201, "\u0120bob": 29202, "\u0120preferably": 29203, "\u0120glitch": 29204, "furt": 29205, "\u0120eg": 29206, "idis": 29207, "CBC": 29208, "\u0120surrendered": 29209, "\u0120testament": 29210, "336": 29211, "uggest": 29212, "\u0120Nil": 29213, "another": 29214, "\u0120pathetic": 29215, "\u0120Donna": 29216, "\u0120218": 29217, "\u0120Avery": 29218, "\u0120whiskey": 29219, "\u0120fixture": 29220, "\u0120Conquest": 29221, "\u0120bets": 29222, "Occ": 29223, "\u0120Leicester": 29224, "].\"": 29225, "\u0120));": 29226, "\u0120flashes": 29227, "456": 29228, "\u0120masked": 29229, "gebra": 29230, "\u0120computed": 29231, "chel": 29232, "auder": 29233, "\u0120defeats": 29234, "\u0120Liberation": 29235, "\u0120Osama": 29236, "\u0120Vive": 29237, "Changes": 29238, "Channel": 29239, "\u0120tariffs": 29240, "\u0120mage": 29241, "\u0120Sax": 29242, "\u0120inadvertently": 29243, "\u0120CRE": 29244, "\u0120Reaper": 29245, "inky": 29246, "grading": 29247, "\u0120stereotyp": 29248, "\u0120curl": 29249, "\u0120FANT": 29250, "\u0120frameworks": 29251, "Mom": 29252, "\u0120Anch": 29253, "\u0120flavour": 29254, "carbon": 29255, "\u0120permitting": 29256, "letcher": 29257, "\u0120Mozilla": 29258, "\u0120Parking": 29259, "\u0120Champ": 29260, "Scroll": 29261, "\u0120murderer": 29262, "\u0120rested": 29263, "\u0120owes": 29264, "\u0120Poss": 29265, "ADD": 29266, "IFF": 29267, "resolution": 29268, "\u0120Mining": 29269, "\u0120comparative": 29270, "Dim": 29271, "\u0120neighbouring": 29272, "\u0120AST": 29273, "\u0120Toxic": 29274, "\u0120biases": 29275, "\u0120gunfire": 29276, "urous": 29277, "\u0120Moment": 29278, "1983": 29279, "\u0120pervasive": 29280, "ttp": 29281, "\u0120Normally": 29282, "rir": 29283, "Sarah": 29284, "\u0120Albany": 29285, "\u0120unsett": 29286, "\u0120SMS": 29287, "ipers": 29288, "layer": 29289, "\u0120Whites": 29290, "uple": 29291, "\u0120turbo": 29292, "\u0120Leeds": 29293, "\u0120thats": 29294, "\u0120Miner": 29295, "MER": 29296, "\u0120Reign": 29297, "\u0120perme": 29298, "\u0120Blitz": 29299, "\u01201934": 29300, "\u0120intimidating": 29301, "tube": 29302, "\u0120eccentric": 29303, "abolic": 29304, "boxes": 29305, "\u0120Associates": 29306, "votes": 29307, "\u0120simulate": 29308, "umbo": 29309, "astery": 29310, "\u0120shipments": 29311, "FFFF": 29312, "anth": 29313, "\u0120seasoned": 29314, "\u0120experimentation": 29315, "\u00e2\u0138\u0142": 29316, "laws": 29317, "Meet": 29318, "iddles": 29319, "antics": 29320, "Rating": 29321, "ISIS": 29322, "hift": 29323, "\u0120fronts": 29324, "buf": 29325, "017": 29326, "\u0120unatt": 29327, "\u0120Dil": 29328, "leases": 29329, "\u0120Gardens": 29330, "777": 29331, "touch": 29332, "vell": 29333, "458": 29334, "\u0120=====": 29335, "saving": 29336, "\u0120erosion": 29337, "\u0120Quin": 29338, "\u0120earns": 29339, "\u0120accomplishment": 29340, "\u0120Wei": 29341, "\u0120<[": 29342, "_____": 29343, "\u0120irrig": 29344, "\u0120Teddy": 29345, "\u0120conquered": 29346, "\u0120Armored": 29347, "\u0120asserts": 29348, "\u0120manipulating": 29349, "r\u00c3\u00a9": 29350, "\u0120transcripts": 29351, "Gallery": 29352, "\u0120plotting": 29353, "Neil": 29354, "\u0120betrayal": 29355, "loader": 29356, "\u0120Sul": 29357, "\u0120displacement": 29358, "\u0120royalty": 29359, "\u0120WI": 29360, "heit": 29361, "\u0120Devices": 29362, "allel": 29363, "\u0120municipalities": 29364, "\u0120canal": 29365, "Stars": 29366, "\u0120UAE": 29367, "\u0120\"\u00e2\u0122\u00a6": 29368, "\u0120CU": 29369, "above": 29370, "\u0120resonance": 29371, "\u0120guiActiveUn": 29372, "added": 29373, "\u0120Braves": 29374, "\u0120Ibn": 29375, "\u0120hereby": 29376, "\u0120BRE": 29377, "\u0120shareholder": 29378, "\u0120Hir": 29379, "\u0120Ji": 29380, "\u0120strangely": 29381, "\u0120admired": 29382, "\u0120plight": 29383, "\u0120bachelor": 29384, "\u0120Pole": 29385, "ciplinary": 29386, "Tony": 29387, "\u0120Armenian": 29388, "\u0120unman": 29389, "\u0120Zionist": 29390, "Stage": 29391, "iscover": 29392, "\u0120automotive": 29393, "\u0120sidelines": 29394, "\u0120slick": 29395, "\u0120Renaissance": 29396, "\u0120FUN": 29397, "Images": 29398, "\u0120Haj": 29399, "\u0120ping": 29400, "\u0120shortcut": 29401, "\u0120Blvd": 29402, "\u0120Looks": 29403, "\u0120bursts": 29404, "\u0120clamp": 29405, "\u0120mish": 29406, "\u0120sorting": 29407, "\u0120patriot": 29408, "\u0120correctness": 29409, "\u0120Scandinav": 29410, "\u0120Cavaliers": 29411, "python": 29412, "azar": 29413, "\u0120375": 29414, "\u0120Jaune": 29415, "409": 29416, "\u0120detrimental": 29417, "\u0120stabbing": 29418, "\u0120poisoned": 29419, "\u0120fountain": 29420, "ocent": 29421, "orst": 29422, "\u0120Mari": 29423, "\u0120rains": 29424, "\u0120Overs": 29425, "\u0120Institution": 29426, "udget": 29427, "AMY": 29428, "tale": 29429, "\u0120KR": 29430, "\u0120Prices": 29431, "\u0120headaches": 29432, "\u0120landsl": 29433, "\u0120Aura": 29434, "Bonus": 29435, "\u0120Zhao": 29436, "\u0120Hip": 29437, "\u0120hops": 29438, "\u0120Kurdistan": 29439, "\u0120exploiting": 29440, "ryn": 29441, "\u0120hypocrisy": 29442, "opening": 29443, "\u0120gunshot": 29444, "\u0120wed": 29445, "interstitial": 29446, "Interstitial": 29447, "\u0120amen": 29448, "Breaking": 29449, "\u0120marketed": 29450, "Wire": 29451, "\u0120Crowd": 29452, "Continue": 29453, "\u0120Known": 29454, "\u0120Effective": 29455, "orean": 29456, "izons": 29457, "Joseph": 29458, "\u0120escalation": 29459, "username": 29460, "\u0120curtain": 29461, "ATES": 29462, "\u0120PAR": 29463, "\u0120Miy": 29464, "\u0120counterfe": 29465, "lene": 29466, "\u0120contenders": 29467, "daily": 29468, "\u0120Asc": 29469, "\u0120Phillip": 29470, "mostly": 29471, "\u0120filename": 29472, "hene": 29473, "\u0120resembling": 29474, "\u0120staging": 29475, "\u0120Chloe": 29476, "\u0120wiring": 29477, "Hon": 29478, "\u0120Renew": 29479, "ottage": 29480, "\u0120Hybrid": 29481, "much": 29482, "\u0120strokes": 29483, "\u0120policymakers": 29484, "APTER": 29485, "\u0120Arkham": 29486, "plot": 29487, "\u0120assistants": 29488, "\u0120deport": 29489, "\u0120Sega": 29490, "\u0120influenza": 29491, "\u0120Cursed": 29492, "\u0120Kobe": 29493, "\u0120skinny": 29494, "Provider": 29495, "\u0120Rip": 29496, "\u0120incremental": 29497, "products": 29498, "BF": 29499, "\u0120dome": 29500, "\u0120Credits": 29501, "\u0120losers": 29502, "ints": 29503, "\u0120Betty": 29504, "\u0120Talent": 29505, "\u0120DAM": 29506, "Lv": 29507, "Ess": 29508, "\u0120dens": 29509, "temp": 29510, "Judge": 29511, "odic": 29512, "\u0120'(": 29513, "URES": 29514, "etsk": 29515, "VO": 29516, "\u0120retrieved": 29517, "\u0120architects": 29518, "\u00d9\u0129": 29519, "\u0120ethic": 29520, "\u0120Secondary": 29521, "stocks": 29522, "adia": 29523, "\u0120325": 29524, "\u0120Opinion": 29525, "\u0120simultaneous": 29526, "\u0120dizz": 29527, "ulp": 29528, "\u0120smuggling": 29529, "ippery": 29530, "Random": 29531, "facing": 29532, "\u0120Das": 29533, "\u0120stockp": 29534, "\u0120disclosures": 29535, "pointer": 29536, "\u0120coral": 29537, "\u0120Selection": 29538, "\u0120Pike": 29539, "ivalent": 29540, "\u0120ruthless": 29541, "\u0120Rim": 29542, "\u0120ensuing": 29543, "\u0120Experiment": 29544, "\u0120congressman": 29545, "\u0120believer": 29546, "\u0120unspecified": 29547, "\u0120Mord": 29548, "\u0120knowledgeable": 29549, "\u0120VERY": 29550, "TX": 29551, "\u0120straps": 29552, "\u0120turf": 29553, "apeshifter": 29554, "\u0120marital": 29555, "\u0120flock": 29556, "\u00e3\u0123\u0128": 29557, "263": 29558, "AMES": 29559, "\u0120Opposition": 29560, "\u0120treasures": 29561, "\u0120GOD": 29562, "\u0120modeled": 29563, "\u0120WORLD": 29564, "\u0120([": 29565, "\u0120Usage": 29566, "HF": 29567, "\u0120$(": 29568, "ussed": 29569, "\u0120pioneer": 29570, "Eight": 29571, "parse": 29572, "bread": 29573, "ritz": 29574, "\u0120Miranda": 29575, "\u0120Kant": 29576, "++)": 29577, "oren": 29578, "\u0120provoked": 29579, "\u0120breeds": 29580, "\u0120Includes": 29581, "\u0120Pastebin": 29582, "\u0120Flip": 29583, "Java": 29584, "\u0120brink": 29585, "\u0120rumored": 29586, "\u0120unseen": 29587, "\u0120garnered": 29588, "\u0120Defin": 29589, "alted": 29590, "\u0120tattoos": 29591, "\u0120hesitation": 29592, "isitions": 29593, "\u0120Weaver": 29594, "\u0120Reporting": 29595, "\u0120therapies": 29596, "\u0120consultants": 29597, "\u0120residual": 29598, "\u0120Mali": 29599, "\u0120Roma": 29600, "iago": 29601, "\u0120Residents": 29602, "ubi": 29603, "\u0120remedies": 29604, "\u0120adaptive": 29605, "\u0120Alive": 29606, "\u0120Barcl": 29607, "\u0120wallets": 29608, "crypt": 29609, "etermination": 29610, "\u0120Pelosi": 29611, "\u0120slipping": 29612, "otonin": 29613, "\u0120alliances": 29614, "patrick": 29615, "iris": 29616, "\u0120orth": 29617, "\u0120Perkins": 29618, "\u0120DeV": 29619, "\u0120Gets": 29620, "\u0120drying": 29621, "gee": 29622, "forest": 29623, "\u0120Forget": 29624, "orem": 29625, "339": 29626, "\u0120vaguely": 29627, "\u0120Dion": 29628, "\u0120Porn": 29629, "\u0120HOW": 29630, "\u0120pneum": 29631, "\u0120rubble": 29632, "\u0120Taste": 29633, "encia": 29634, "\u0120Gel": 29635, "\u0120dst": 29636, "\u0120245": 29637, "\u0120Morocco": 29638, "inflamm": 29639, "\u0120Twins": 29640, "\u0120bots": 29641, "daughter": 29642, "\u0120Balk": 29643, "\u0120brethren": 29644, "\u0120logos": 29645, "\u0120gobl": 29646, "fps": 29647, "\u0120subdivision": 29648, "\u0120pawn": 29649, "\u0120squeezed": 29650, "\u0120morale": 29651, "\u0120DW": 29652, "'\"": 29653, "\u0120knot": 29654, "ooky": 29655, "\u0120divisive": 29656, "\u0120boosted": 29657, "chy": 29658, "\u00e3\u0125\u0132": 29659, "ifact": 29660, "\u0120newcomers": 29661, "\u0120Wrestling": 29662, "\u0120scouts": 29663, "wolves": 29664, "Rat": 29665, "\u0120nineteenth": 29666, "\u0120Osborne": 29667, "Stats": 29668, "\u0120empowered": 29669, "\u0120psychopath": 29670, "\u0120OEM": 29671, "uggage": 29672, "\u0120PK": 29673, "\u0120Mohammad": 29674, "Pak": 29675, "\u0120anarchists": 29676, "\u0120Extract": 29677, "esthes": 29678, "\u0120Stockholm": 29679, "loo": 29680, "\u0120Graph": 29681, "\u0120deploying": 29682, "\u0120Stranger": 29683, "\u0120Mold": 29684, "\u0120staffer": 29685, "\u0120discounted": 29686, "uckle": 29687, "please": 29688, "\u0120Landing": 29689, "\u00c3\u0143a": 29690, "\u0120193": 29691, "\u0120ante": 29692, "\u0120repetition": 29693, "\u0120+/-": 29694, "\u0120parody": 29695, "\u0120lively": 29696, "AAA": 29697, "\u0120Horus": 29698, "\u0120pits": 29699, "inders": 29700, "LOC": 29701, "\u0120Venice": 29702, "406": 29703, "\u0120Discover": 29704, "\u00e2\u0128": 29705, "ellectual": 29706, "\u0120pens": 29707, "\u0120eyel": 29708, "iguous": 29709, "Impl": 29710, "\u0120joking": 29711, "\u0120inval": 29712, "\u0120Belfast": 29713, "\u0120creditors": 29714, "\u0120Skywalker": 29715, "ovsky": 29716, "\u0120ceasefire": 29717, "\u0120seals": 29718, "isoft": 29719, ")).": 29720, "\u0120Felix": 29721, "ITS": 29722, "\u0120tresp": 29723, "\u0120Blockchain": 29724, "eware": 29725, "\u0120Schwar": 29726, "enne": 29727, "mounted": 29728, "\u0120Beacon": 29729, "lesh": 29730, "\u0120immensely": 29731, "\u0120cheering": 29732, "Employ": 29733, "scene": 29734, "ishly": 29735, "atchewan": 29736, "\u0120Nicolas": 29737, "\u0120drained": 29738, "\u0120Exit": 29739, "\u0120Azerb": 29740, "jun": 29741, "\u0120floated": 29742, "uania": 29743, "Deep": 29744, "\u0120superv": 29745, "\u0120mystical": 29746, "\u0120Dollar": 29747, "\u0120Apostle": 29748, "\u0120REL": 29749, "\u0120Provided": 29750, "\u0120Bucks": 29751, "\u00e3\u0125\u00b4": 29752, "cutting": 29753, "\u0120enhancements": 29754, "\u0120Penguins": 29755, "\u0120Isaiah": 29756, "\u0120jerk": 29757, "\u0120Wyn": 29758, "\u0120stalled": 29759, "\u0120cryptocurrencies": 29760, "\u0120Roland": 29761, "single": 29762, "\u0120lumin": 29763, "\u0120Fellow": 29764, "\u0120Capacity": 29765, "\u0120Kazakh": 29766, "WN": 29767, "\u0120financed": 29768, "389": 29769, "\u0120tid": 29770, "\u0120collusion": 29771, "\u0120Myr": 29772, "\u00ee\u0122": 29773, "Senator": 29774, "\u0120pediatric": 29775, "\u0120neatly": 29776, "\u0120sandwiches": 29777, "\u0120Architecture": 29778, "\u0120tucked": 29779, "\u0120balcony": 29780, "\u0120earthquakes": 29781, "quire": 29782, "Future": 29783, "\u0120hefty": 29784, "\u00e9\u0139": 29785, "\u0120specializes": 29786, "\u0120stresses": 29787, "\u0120sender": 29788, "\u0120misunderstanding": 29789, "\u0120epile": 29790, "\u0120provoke": 29791, "\u0120Colors": 29792, "\u0120dismay": 29793, "uko": 29794, "[_": 29795, "586": 29796, "neutral": 29797, "\u0120donating": 29798, "\u0120Randall": 29799, "Multi": 29800, "\u0120conveniently": 29801, "\u0120Sung": 29802, "\u0120Coca": 29803, "\u0120tents": 29804, "\u0120Acceler": 29805, "\u0120partnered": 29806, "272": 29807, "irming": 29808, "\u0120BAS": 29809, "sometimes": 29810, "\u0120objected": 29811, "ubric": 29812, "posed": 29813, "LCS": 29814, "grass": 29815, "\u0120attributable": 29816, "VIS": 29817, "Israeli": 29818, "\u0120repeats": 29819, "\u0120RM": 29820, "vag": 29821, "uta": 29822, "inous": 29823, "\u0120inert": 29824, "\u0120Miguel": 29825, "\u00e6\u0143": 29826, "\u0120Hawaiian": 29827, "Board": 29828, "\u0120artific": 29829, "\u0120Azerbai": 29830, "asio": 29831, "\u0120Rent": 29832, "AIN": 29833, "\u0120appliances": 29834, "\u0120nationality": 29835, "\u0120asshole": 29836, "\u0120Neb": 29837, "\u0120notch": 29838, "hani": 29839, "\u0120Bride": 29840, "Availability": 29841, "\u0120intercepted": 29842, "\u0120continental": 29843, "\u0120swelling": 29844, "\u0120Perspect": 29845, "bies": 29846, ".<": 29847, "ithmetic": 29848, "\u0120Lara": 29849, "\u0120tempting": 29850, "addr": 29851, "\u0120overseeing": 29852, "clad": 29853, "\u0120DV": 29854, "\u0120Gingrich": 29855, "\u0120mun": 29856, "\u0120Appropri": 29857, "\u0120alterations": 29858, "\u0120Patreon": 29859, "\u0120havoc": 29860, "\u0120disciplines": 29861, "\u0120notoriously": 29862, "akuya": 29863, "ieri": 29864, "?).": 29865, "\u0120Went": 29866, "\u0120silicon": 29867, "\u0120tremb": 29868, "Container": 29869, "Known": 29870, "\u0120mortar": 29871, "este": 29872, "icka": 29873, "Arthur": 29874, "\u0120Previously": 29875, "\u0120Marty": 29876, "\u0120sparse": 29877, "gins": 29878, "\u0120inward": 29879, "\u0120Participant": 29880, "Copy": 29881, "\u0120Misc": 29882, "\u0120antibiotic": 29883, "\u0120Retro": 29884, "\u0120elusive": 29885, "\u0120assail": 29886, "\u0120Battalion": 29887, "\u0120Bought": 29888, "\u0120diminish": 29889, "\u0120Europa": 29890, "session": 29891, "\u0120Dangerous": 29892, "iesel": 29893, "\u0120disbelief": 29894, "\u0120blasts": 29895, "extreme": 29896, "\u0120Boyd": 29897, "\u0120Projects": 29898, "\u0120Guys": 29899, "\u0120undergone": 29900, "\u0120grill": 29901, "\u0120Dwight": 29902, "\u0120197": 29903, "USER": 29904, "\u0120filesystem": 29905, "\u0120clocks": 29906, "Taylor": 29907, "\u0120wrapper": 29908, "\u0120folding": 29909, "ousand": 29910, "\u0120Philippine": 29911, "ATIONAL": 29912, "\u0120Perth": 29913, "\u0120ashes": 29914, "\u0120accumulate": 29915, "\u0120Gateway": 29916, "Shop": 29917, "orkshire": 29918, "Han": 29919, "\u0120Barrel": 29920, "\u0120Leh": 29921, "\u0120XV": 29922, "\u0120whim": 29923, "\u0120repo": 29924, "\u0120CG": 29925, "\u0120Mam": 29926, "\u0120incorporating": 29927, "\u0120bailout": 29928, "\u0120linguistic": 29929, "\u0120disinteg": 29930, "CLE": 29931, "\u0120cinematic": 29932, "\u0120Fiber": 29933, "Syn": 29934, "ilion": 29935, "\u0120Compos": 29936, "chens": 29937, "\u0120neoc": 29938, "\u0120boiled": 29939, "FINE": 29940, "ono": 29941, "uncle": 29942, "iken": 29943, "\u0120BM": 29944, "\u00ce\u00b9": 29945, "\u0120receipts": 29946, "\u0120disposed": 29947, "\u0120Thirty": 29948, "\u0120Rough": 29949, "\u0120ABS": 29950, "\u0120notwithstanding": 29951, "ollen": 29952, "#$": 29953, "\u0120unreliable": 29954, "\u0120bloom": 29955, "\u0120mediocre": 29956, "\u0120tram": 29957, "\u0120Tasman": 29958, "\u0120shakes": 29959, "\u0120manifesto": 29960, "\u0120MW": 29961, "\u0120satisfactory": 29962, "\u0120shores": 29963, "\u0120computation": 29964, "\u0120assertions": 29965, "ormons": 29966, "arag": 29967, "abit": 29968, "Democrats": 29969, "\u0120Loot": 29970, "\u0120Volks": 29971, "haired": 29972, "\u0120gravitational": 29973, "Sing": 29974, "\u0120Miz": 29975, "\u0120throttle": 29976, "\u0120tyranny": 29977, "\u0120Views": 29978, "\u0120robber": 29979, "\u0120Minority": 29980, "\u0120shrine": 29981, "scope": 29982, "purpose": 29983, "\u0120nucleus": 29984, "ourcing": 29985, "\u0120USDA": 29986, "\u0120DHS": 29987, "wra": 29988, "\u0120Bowie": 29989, "Scale": 29990, "\u0120BEL": 29991, "xi": 29992, "Iter": 29993, "\u0120(),": 29994, "wright": 29995, "\u0120sailors": 29996, "oused": 29997, "NASA": 29998, "\u0120Proof": 29999, "\u0120Mineral": 30000, "token": 30001, "\u0120FD": 30002, "Rew": 30003, "\u0120ell": 30004, "630": 30005, "\u0120chancellor": 30006, "\u0120Gos": 30007, "\u0120amounted": 30008, "\u0120Recre": 30009, "omez": 30010, "\u0120Optim": 30011, "\u0120Olive": 30012, "\u0120tracker": 30013, "owler": 30014, "\u0120Unique": 30015, "Root": 30016, "\u0120maritime": 30017, "\u0120Quran": 30018, "\u0120Adapt": 30019, "\u0120ecosystems": 30020, "\u0120Repeat": 30021, "\u0120Soy": 30022, "\u0120IMP": 30023, "\u0120graduating": 30024, "andem": 30025, "Pur": 30026, "\u0120Reset": 30027, "\u0120Trick": 30028, "\u0120Philly": 30029, "\u0120Tue": 30030, "\u0120Malaysian": 30031, "\u0120climax": 30032, "\u0120bury": 30033, "\u0120conspic": 30034, "\u0120Southampton": 30035, "\u0120Flowers": 30036, "\u0120escorted": 30037, "\u0120Educational": 30038, "\u0120IRC": 30039, "\u0120brutally": 30040, "eating": 30041, "\u0120pillar": 30042, "\u0120Sang": 30043, "\u0120Jude": 30044, "arling": 30045, "\u0120Amnesty": 30046, "\u0120reminding": 30047, "\u0120Administrative": 30048, "hesda": 30049, "\u0120flashed": 30050, "\u0120PBS": 30051, "perate": 30052, "feature": 30053, "\u0120swipe": 30054, "\u0120graves": 30055, "oultry": 30056, "261": 30057, "breaks": 30058, "\u0120Guer": 30059, "\u0120shrimp": 30060, "\u0120Voting": 30061, "quist": 30062, "\u0120analytical": 30063, "\u0120tablespoons": 30064, "\u0120SOU": 30065, "\u0120researched": 30066, "\u0120disrupted": 30067, "\u0120jour": 30068, "\u0120replica": 30069, "\u0120cartoons": 30070, "bians": 30071, "})": 30072, "copy": 30073, "Got": 30074, "ouched": 30075, "PUT": 30076, "\u0120swarm": 30077, "notations": 30078, "said": 30079, "\u0120rebuilt": 30080, "\u0120collaborate": 30081, "\u0120raging": 30082, "\u0120nar": 30083, "\u0120demographics": 30084, "\u0120DDR": 30085, "\u0120distrust": 30086, "ossier": 30087, "\u0120Kro": 30088, "\u0120pumpkin": 30089, "\u0120regrets": 30090, "\u0120fatalities": 30091, "\u0120Lens": 30092, "\u0120Ole": 30093, "pd": 30094, "\u0120puppet": 30095, "\u0120Outlook": 30096, "\u0120Stam": 30097, "Ol": 30098, "Fair": 30099, "UU": 30100, "\u0120rewritten": 30101, "\u00c4\u00b1": 30102, "\u0120fascinated": 30103, "\u0120vectors": 30104, "\u0120tribunal": 30105, "uay": 30106, "\u0120Mats": 30107, "\u0120Coins": 30108, "[[": 30109, "\u0120181": 30110, "\u0120renders": 30111, "\u0120Kaepernick": 30112, "\u0120espionage": 30113, "\u0120summ": 30114, "\u0120ditch": 30115, "Account": 30116, "\u0120spreadsheet": 30117, "\u0120mutant": 30118, "past": 30119, "407": 30120, "\u0120dye": 30121, "\u0120initiation": 30122, "\u01204000": 30123, "\u0120punishable": 30124, "\u0120thinner": 30125, "\u0120Khal": 30126, "\u0120intermedi": 30127, "Dun": 30128, "\u0120Gotham": 30129, "\u0120eagerly": 30130, "\u0120vaginal": 30131, "powers": 30132, "VW": 30133, "\u0120WATCHED": 30134, "\u0120predator": 30135, "amsung": 30136, "\u0120disparity": 30137, "\u0120[*": 30138, "\u0120amph": 30139, "\u0120outskirts": 30140, "\u0120Spirits": 30141, "\u0120skeletal": 30142, "\u00d0\u00bb": 30143, "\u0120Rear": 30144, "\u0120issuance": 30145, "\u0120Logic": 30146, "released": 30147, "ZZ": 30148, "\u0120Bound": 30149, "Entry": 30150, "\u0120exits": 30151, "isol": 30152, "\u0120Founder": 30153, "\u0120wre": 30154, "\u0120Greenland": 30155, "\u0120MMO": 30156, "taker": 30157, "INC": 30158, "\u00e3\u0123\u00be": 30159, "\u0120hourly": 30160, "henko": 30161, "\u0120fantasies": 30162, "\u0120disob": 30163, "\u0120demolition": 30164, "\u00e3\u0125\u012d": 30165, "\u0120enlisted": 30166, "ratulations": 30167, "\u0120misguided": 30168, "\u0120ensured": 30169, "\u0120discouraged": 30170, "mort": 30171, "\u0120flank": 30172, "\u0120cess": 30173, "\u0120reacts": 30174, "\u0120Sere": 30175, "sensitive": 30176, "\u0120Serpent": 30177, "assad": 30178, "\u0120247": 30179, "\u0120calmly": 30180, "busters": 30181, "\u0120bleed": 30182, "\u0120Stro": 30183, "\u0120amusement": 30184, "\u0120Antarctica": 30185, "\u0120scept": 30186, "\u0120Gaw": 30187, "aq": 30188, "asonic": 30189, "\u0120sprawling": 30190, "native": 30191, "aturated": 30192, "\u0120Battlefield": 30193, "IVERS": 30194, "EB": 30195, "\u0120Gems": 30196, "\u0120Northwestern": 30197, "\u0120Films": 30198, "\u0120Automatic": 30199, "\u0120apprehend": 30200, "\u00e3\u0123\u00a8": 30201, "\u0120guiName": 30202, "\u0120backend": 30203, "\u0120evidenced": 30204, "geant": 30205, "012": 30206, "\u0120Siege": 30207, "\u0120externalTo": 30208, "\u0120unfocusedRange": 30209, "\u0120guiActiveUnfocused": 30210, "\u0120guiIcon": 30211, "\u0120externalToEVA": 30212, "\u0120externalToEVAOnly": 30213, "Fri": 30214, "chard": 30215, "enaries": 30216, "\u0120chiefs": 30217, "\u0120cf": 30218, "\u0120HUD": 30219, "\u0120corrobor": 30220, "\u0120dB": 30221, "\u0120Taken": 30222, "\u0120Patricia": 30223, "rail": 30224, "\u0120Charm": 30225, "\u0120Libertarian": 30226, "rieve": 30227, "Personal": 30228, "\u0120OUR": 30229, "geries": 30230, "\u0120dumping": 30231, "\u0120neurological": 30232, "itimate": 30233, "\u0120Clintons": 30234, "rafted": 30235, "\u0120Molly": 30236, "\u0120terminals": 30237, "register": 30238, "\u0120flare": 30239, "\u0120encoded": 30240, "\u0120autopsy": 30241, "pel": 30242, "machine": 30243, "\u0120exemptions": 30244, "\u0120Royals": 30245, "distance": 30246, "\u0120drafts": 30247, "\u0120lame": 30248, "\u0120Cunning": 30249, "\u0120spouses": 30250, "\u0120Markets": 30251, "\u0120Carrier": 30252, "\u0120implying": 30253, "\u0120Yak": 30254, "sid": 30255, "\u0120loser": 30256, "\u0120vigilant": 30257, "\u0120impeachment": 30258, "\u0120augmented": 30259, "\u0120Employees": 30260, "\u0120unintended": 30261, "ternally": 30262, "\u0120Watt": 30263, "\u0120recognizable": 30264, "essim": 30265, "\u00e6\u013f": 30266, "\u0120coated": 30267, "rha": 30268, "\u0120lieutenant": 30269, "\u0120Legislation": 30270, "published": 30271, "444": 30272, "013": 30273, "\u0120ideally": 30274, "\u0120Password": 30275, "\u0120simplify": 30276, "\u0120Meta": 30277, "\u0120MRI": 30278, "\u0120pleading": 30279, "organized": 30280, "handler": 30281, "\u0120unravel": 30282, "correct": 30283, "\u0120icy": 30284, "\u0120paranoid": 30285, "\u0120passer": 30286, "\u0120inspections": 30287, "ofer": 30288, "\u0120Healthcare": 30289, "283": 30290, "\u0120Brut": 30291, "iola": 30292, "forge": 30293, "\u0120Medieval": 30294, "MSN": 30295, "ievers": 30296, "\u0120Programming": 30297, "\u00e5\u012b": 30298, "\u0120223": 30299, "mu": 30300, "\u0120CLE": 30301, "uga": 30302, "\u0120shoppers": 30303, "\u0120informative": 30304, "\u0120Plans": 30305, "\u0120supplementation": 30306, "\u0120Tests": 30307, "tyard": 30308, "ocytes": 30309, "\u0120Vega": 30310, "\u0120Gujarat": 30311, "ermanent": 30312, "Except": 30313, "\u0120LOT": 30314, "alla": 30315, "\u0120Cumm": 30316, "\u0120Osw": 30317, "\u0120venom": 30318, "\u0120Debt": 30319, "\u0120DOWN": 30320, "\u0120reunion": 30321, "\u0120muc": 30322, "\u0120Relief": 30323, "\u0120geop": 30324, "\u0120\u00f0\u0141\u013a": 30325, "alogue": 30326, "Anth": 30327, "echo": 30328, "\u0120corros": 30329, "\u0120replication": 30330, "\u0120Blazing": 30331, "\u0120Daughter": 30332, "\u0120inflic": 30333, "\u0120Lindsey": 30334, "\u00d9\u012a": 30335, "284": 30336, "Exit": 30337, "\u0120gloom": 30338, "TAIN": 30339, "\u0120undermining": 30340, "\u0120advising": 30341, "hidden": 30342, "\u0120overflow": 30343, "\u0120gor": 30344, "urdue": 30345, "\u0120echoes": 30346, "enhagen": 30347, "\u0120impuls": 30348, "drug": 30349, "cash": 30350, "\u0120async": 30351, "\u0120mirac": 30352, "atts": 30353, "punk": 30354, "\u0120pivot": 30355, "\u0120Legislative": 30356, "\u0120bloggers": 30357, "\u0120Claw": 30358, "sburg": 30359, "dyl": 30360, "\u0120Recommend": 30361, "\u0120verte": 30362, "\u0120prohibiting": 30363, "\u0120Panther": 30364, "Jonathan": 30365, "\u0120omin": 30366, "\u0120hateful": 30367, "281": 30368, "\u0120Orche": 30369, "\u0120Murdoch": 30370, "downs": 30371, "\u0120asymm": 30372, "GER": 30373, "Always": 30374, "\u0120informs": 30375, "\u0120WM": 30376, "\u0120Pony": 30377, "\u0120Appendix": 30378, "\u0120Arlington": 30379, "Jam": 30380, "\u0120medicinal": 30381, "\u0120Slam": 30382, "ITIES": 30383, "\u0120reaff": 30384, "\u0120Ri": 30385, "FG": 30386, "Spring": 30387, "bool": 30388, "\u0120thighs": 30389, "\u0120markings": 30390, "\u0120Raqqa": 30391, "\u0120Lak": 30392, "poll": 30393, "tsky": 30394, "\u0120Morty": 30395, "\u0120Definition": 30396, "\u0120debunk": 30397, "endered": 30398, "\u0120Leone": 30399, "avers": 30400, "\u0120mortgages": 30401, "Apparently": 30402, "Nic": 30403, "haus": 30404, "\u0120Thousands": 30405, "auld": 30406, "\u0120mash": 30407, "shoot": 30408, "\u0120diarr": 30409, "\u0120consciously": 30410, "Hero": 30411, "eas": 30412, "\u0120Naturally": 30413, "\u0120Destroyer": 30414, "\u0120dashboard": 30415, "services": 30416, "Rog": 30417, "\u0120millennials": 30418, "\u0120invade": 30419, "-(": 30420, "\u0120commissions": 30421, "\u0120Auckland": 30422, "\u0120broadcasts": 30423, "\u0120frontal": 30424, "\u0120crank": 30425, "\u0120Historic": 30426, "\u0120rumours": 30427, "CTV": 30428, "\u0120steril": 30429, "\u0120booster": 30430, "rocket": 30431, "\u00e3\u0124\u00bc": 30432, "utsche": 30433, "\u0120PI": 30434, "\u0120233": 30435, "\u0120Producer": 30436, "\u0120Analytics": 30437, "\u0120invaluable": 30438, "\u0120unintention": 30439, "\u0120CY": 30440, "\u0120scrutin": 30441, "\u0120gigg": 30442, "\u0120engulf": 30443, "\u0120proletariat": 30444, "\u0120hacks": 30445, "\u0120Hew": 30446, "arak": 30447, "\u0120Slime": 30448, "ielding": 30449, "agher": 30450, "\u0120Elliot": 30451, "\u0120telecom": 30452, "\u0120219": 30453, "ultan": 30454, "\u0120Arbor": 30455, "\u0120Scouts": 30456, "Ban": 30457, "\u0120lifespan": 30458, "\u0120blasp": 30459, "388": 30460, "\u0120judiciary": 30461, "\u0120Continental": 30462, "asking": 30463, "McC": 30464, "LED": 30465, "\u0120baggage": 30466, "\u0120Sorcerer": 30467, "\u0120remnants": 30468, "\u0120Griffith": 30469, "etsu": 30470, "\u0120Subaru": 30471, "\u0120Personality": 30472, "designed": 30473, "ushima": 30474, "agnar": 30475, "\u0120recoil": 30476, "\u0120passions": 30477, "\\\":": 30478, "\u0120tee": 30479, "\u0120abolition": 30480, "\u0120Creating": 30481, "jac": 30482, "\u0120194": 30483, "019": 30484, "\u0120pillars": 30485, "riched": 30486, "/\"": 30487, "tk": 30488, "\u0120livelihood": 30489, "\u0120roasted": 30490, "ahon": 30491, "\u0120Hutch": 30492, "assert": 30493, "\u0120dividend": 30494, "\u0120knit": 30495, "\u0120daunting": 30496, "\u0120disturbance": 30497, "\u0120shale": 30498, "\u0120cultivated": 30499, "\u0120refrigerator": 30500, "LB": 30501, "\u0120NET": 30502, "\u0120commercials": 30503, "\u0120thinkers": 30504, "455": 30505, "\u0120chop": 30506, "Broad": 30507, "\u0120suspicions": 30508, "\u0120tagged": 30509, "lifting": 30510, "\u0120stylish": 30511, "\u0120Shields": 30512, "Shortly": 30513, "\u0120tails": 30514, "Auth": 30515, "STE": 30516, "\u0120GAME": 30517, "\u0120seism": 30518, "\u0120Kis": 30519, "ologne": 30520, "\u0120cowork": 30521, "\u0120forcibly": 30522, "\u0120thyroid": 30523, "\u0120PB": 30524, "ANE": 30525, "married": 30526, "horse": 30527, "\u0120polymer": 30528, "\u0120Chal": 30529, "odor": 30530, "DEBUG": 30531, "\u0120Context": 30532, "\u0120bliss": 30533, "\u0120pinpoint": 30534, "\u0120Mathemat": 30535, "legram": 30536, "\u0120Weekend": 30537, "\u0120labelled": 30538, "\u0120bart": 30539, "itles": 30540, "\u0120estrogen": 30541, "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 30542, "\"'": 30543, "\u0120visibly": 30544, "\u0120outsider": 30545, "aida": 30546, "Area": 30547, "\u0120dissemin": 30548, "\u0120dishonest": 30549, "\u0120Closed": 30550, "\u0120Bulletin": 30551, "\u0120Ramsey": 30552, "sword": 30553, "\u0120XI": 30554, "ourced": 30555, "Same": 30556, "346": 30557, "\u0120Repe": 30558, "\u0120Kou": 30559, "cake": 30560, "emis": 30561, "Cache": 30562, "\u0120Meaning": 30563, "\u0120Enlight": 30564, "onomy": 30565, "\u0120manifestation": 30566, "sworth": 30567, "Jay": 30568, "\u0120chore": 30569, "\u00c3\u00b6r": 30570, "Dream": 30571, "\u0120sanctioned": 30572, "\u0120culturally": 30573, "\u0120Ara": 30574, "Nav": 30575, "\u0120theological": 30576, "\u0120strut": 30577, "\u0120VO": 30578, "\u0120Handbook": 30579, "\u0120constructing": 30580, "\u0120\u00c2\u00b6": 30581, "\u0120Benefits": 30582, "\u0120Psychological": 30583, "sac": 30584, "\u00e5\u00b8": 30585, "policy": 30586, "\u0120Matters": 30587, "\u0120Reported": 30588, "\u0120Byte": 30589, "\u0120vitro": 30590, "\u0120Maiden": 30591, "\u0120lam": 30592, "\u0120Jennings": 30593, "\u0120garment": 30594, "\u0120Rutgers": 30595, "\u0120Stafford": 30596, "\u0120Wellington": 30597, "\u0120intermitt": 30598, "\u0120npm": 30599, "\u0120ordeal": 30600, "\u0120plugged": 30601, "ooming": 30602, "inished": 30603, "framework": 30604, "\u0120timber": 30605, "\u0120cass": 30606, "\u0120850": 30607, "iless": 30608, "\u0120Redux": 30609, "768": 30610, "Stre": 30611, "\u0120surpassed": 30612, "whel": 30613, "\u0120parallels": 30614, "\u0120veil": 30615, "\u0120GI": 30616, "\u0120REST": 30617, "\u0120readiness": 30618, "sort": 30619, "\u0120modifying": 30620, "\u0120Slate": 30621, "ruff": 30622, "\u0120marble": 30623, "\u0120infrared": 30624, "\u0120auditor": 30625, "\u0120FANTASY": 30626, "\u0120Poverty": 30627, "\u0120SPD": 30628, "\u0120\"(": 30629, "Ky": 30630, "RAY": 30631, "\u0120executions": 30632, "\u0120Beverly": 30633, "\u0120Marxism": 30634, "\u0120Burst": 30635, "\u0120Kali": 30636, "estones": 30637, "Clearly": 30638, "Ell": 30639, "\u00e3\u0123\u00a7": 30640, "\u0120Proceedings": 30641, "Token": 30642, "IFIC": 30643, "\u00c3\u00b1a": 30644, "Central": 30645, "\u0120Haley": 30646, "\u0120Drama": 30647, "\u0120formations": 30648, "ORN": 30649, "Books": 30650, "\u0120dominating": 30651, "\u0120Flyers": 30652, "\u0120Companion": 30653, "\u0120disciplined": 30654, "\u0120Yugoslav": 30655, "\u0120Spells": 30656, "\u0120vengeance": 30657, "\u0120landlords": 30658, "Len": 30659, "\u0120Ogre": 30660, "anoia": 30661, "\u0120piercing": 30662, "\u0120congreg": 30663, "\u0120scorer": 30664, "obia": 30665, "\u0120nickel": 30666, "\u0120Learns": 30667, "\u0120rejo": 30668, "\u0120masterpiece": 30669, "Flash": 30670, "\u0120inhabited": 30671, "\u0120OpenGL": 30672, "\u0120Dud": 30673, "\u0120ICO": 30674, "\u0120arter": 30675, "\u0120plur": 30676, "\u0120mastery": 30677, "\u0120longstanding": 30678, "sted": 30679, "\u0120wines": 30680, "\u0120televised": 30681, "\u0120Shrine": 30682, "\u0120Bayern": 30683, "\u0120\u00e2\u0135\u013a": 30684, "\u0120enclosure": 30685, "john": 30686, "\u0120prophets": 30687, "\u0120Resurrection": 30688, "\u0120Orders": 30689, "\u0120uneven": 30690, "rals": 30691, "\u0120dwind": 30692, "\u0120Lah": 30693, "\u0120Sloven": 30694, "378": 30695, "\u0120insistence": 30696, "affle": 30697, "\u0120Clone": 30698, "\u0120hardship": 30699, "\u0120Congressman": 30700, "\u0120plead": 30701, "\u0120reviewers": 30702, "\u0120cured": 30703, "\u01201935": 30704, "asley": 30705, "fake": 30706, "\u0120Thinking": 30707, "ydia": 30708, "PART": 30709, "\u0120Dota": 30710, "oit": 30711, "\u0120whipped": 30712, "\u0120bouncing": 30713, "\u0120Hispanics": 30714, "comings": 30715, "\u0120cannabin": 30716, "\u0120Chambers": 30717, "\u0120Zack": 30718, "Optional": 30719, "\u0120coats": 30720, "\u0120prowess": 30721, "\u0120Norton": 30722, "\u0120plainly": 30723, "\u0120freight": 30724, "\u0120inhibition": 30725, "\u0120clam": 30726, "\u0120303": 30727, "kef": 30728, "aleigh": 30729, "Luke": 30730, "\u0120psycho": 30731, "atorium": 30732, "MED": 30733, "\u0120treaties": 30734, "\u0120indisc": 30735, "\u0120dc": 30736, "OPS": 30737, "\u0120resilient": 30738, "\u0120Interstate": 30739, "\u0120slack": 30740, "\u0120mundane": 30741, "\u0120establishes": 30742, "359": 30743, "\u0120strained": 30744, "\u0120nond": 30745, "Sus": 30746, "\u0120caste": 30747, "arate": 30748, "ieving": 30749, "\u0120unfairly": 30750, "\u0120parser": 30751, "onial": 30752, "ursive": 30753, "Via": 30754, "\u0120Otto": 30755, "\u0120Authorities": 30756, "stroke": 30757, "KR": 30758, "\u0120Mercy": 30759, "\u0120furnished": 30760, "\u0120outset": 30761, "\u0120metic": 30762, "1982": 30763, "olithic": 30764, "\u0120Tent": 30765, "ogical": 30766, "\u0120Aircraft": 30767, "\u0120hides": 30768, "\u0120Became": 30769, "\u0120educators": 30770, "reaching": 30771, "\u0120volatility": 30772, "\u0120toddler": 30773, "\u0120NASCAR": 30774, "\u0120Twelve": 30775, "\u0120Highlights": 30776, "\u0120grape": 30777, "\u0120splits": 30778, "\u0120peasant": 30779, "\u0120reneg": 30780, "\u0120MSI": 30781, "Temp": 30782, "stars": 30783, "\u0120trek": 30784, "\u0120Hyde": 30785, "binding": 30786, "\u0120realism": 30787, "\u0120oxide": 30788, "\u0120Hos": 30789, "\u0120mounts": 30790, "\u0120biting": 30791, "\u0120collapsing": 30792, "\u0120postal": 30793, "\u0120museums": 30794, "\u0120detached": 30795, "\u0120respecting": 30796, "\u0120monopol": 30797, "\u0120workflow": 30798, "\u0120Cake": 30799, "Template": 30800, "\u0120Organisation": 30801, "\u0120persistence": 30802, "369": 30803, "Coming": 30804, "Brad": 30805, "\u0120redundant": 30806, "\u0120GTA": 30807, "\u0120bending": 30808, "\u0120revoked": 30809, "\u0120offending": 30810, "\u0120framing": 30811, "\u0120printf": 30812, "Commun": 30813, "members": 30814, "Outside": 30815, "\u0120construed": 30816, "\u0120coded": 30817, "FORE": 30818, "\u0120chast": 30819, "Chat": 30820, "Indian": 30821, "\u0120Yard": 30822, "?!\"": 30823, "\u0120Ports": 30824, "\u0120Xavier": 30825, "\u0120RET": 30826, "'.\"": 30827, "\u0120Boat": 30828, "ivated": 30829, "icht": 30830, "umerable": 30831, "Ds": 30832, "\u0120Dunn": 30833, "\u0120coffin": 30834, "\u0120securely": 30835, "\u0120Raptors": 30836, "\u0120Bes": 30837, "Installation": 30838, "\u0120inception": 30839, "\u0120Healthy": 30840, "endants": 30841, "\u0120psychologists": 30842, "\u0120Sheikh": 30843, "cultural": 30844, "\u0120BlackBerry": 30845, "shift": 30846, "Fred": 30847, "oche": 30848, "\u0120cakes": 30849, "\u0120SEO": 30850, "\u0120Gian": 30851, "\u0120Asians": 30852, "ogging": 30853, "element": 30854, "\u0120pundits": 30855, "\u0120Vaugh": 30856, "\u0120Gavin": 30857, "\u0120hitter": 30858, "\u0120drowned": 30859, "\u0120chalk": 30860, "\u0120Zika": 30861, "\u0120measles": 30862, "802": 30863, "\u00e2\u0122\u00a6..": 30864, "\u0120AWS": 30865, "]\"": 30866, "\u0120distort": 30867, "\u0120Mast": 30868, "\u0120antibodies": 30869, "\u0120Mash": 30870, "Memory": 30871, "\u0120Uganda": 30872, "\u0120Prob": 30873, "\u0120vomiting": 30874, "\u0120Turns": 30875, "\u0120occupying": 30876, "\u0120evasion": 30877, "\u0120Therapy": 30878, "\u0120promo": 30879, "\u0120electr": 30880, "\u0120blueprint": 30881, "\u0120Dre": 30882, "priced": 30883, "\u0120Depot": 30884, "\u0120alleviate": 30885, "\u0120Somali": 30886, "marg": 30887, "nine": 30888, "\u0120nostalgia": 30889, "\u0120Shepherd": 30890, "\u0120cavalry": 30891, "\u0120torped": 30892, "\u0120Bloody": 30893, "xb": 30894, "\u0120sank": 30895, "\u0120goalt": 30896, "reportprint": 30897, "embedreportprint": 30898, "cloneembedreportprint": 30899, "\u0120Initially": 30900, "\u0120Fischer": 30901, "\u0120noteworthy": 30902, "cern": 30903, "\u0120inefficient": 30904, "rawdownload": 30905, "rawdownloadcloneembedreportprint": 30906, "cation": 30907, "\u0120Dynasty": 30908, "lag": 30909, "DES": 30910, "\u0120distinctly": 30911, "\u0120Estonia": 30912, "\u0120openness": 30913, "\u0120gossip": 30914, "ruck": 30915, "Width": 30916, "\u0120Ibrahim": 30917, "\u0120petroleum": 30918, "\u0120avatar": 30919, "\u0120Hed": 30920, "atha": 30921, "\u0120Hogwarts": 30922, "\u0120caves": 30923, "678": 30924, "\u0120safeguard": 30925, "\u0120Mog": 30926, "isson": 30927, "\u0120Durham": 30928, "slaught": 30929, "\u0120Graduate": 30930, "\u0120subconscious": 30931, "\u0120Excellent": 30932, "\u0120Dum": 30933, "-----": 30934, "\u0120piles": 30935, "\u0120WORK": 30936, "\u0120Garn": 30937, "\u0120Fol": 30938, "\u0120ATM": 30939, "\u0120avoids": 30940, "\u0120Tul": 30941, "\u0120bleak": 30942, "ELY": 30943, "ivist": 30944, "lightly": 30945, "Pers": 30946, "\u0120Dob": 30947, "\u0120LS": 30948, "\u0120insanity": 30949, "\u00ce\u00b5": 30950, "atalie": 30951, "Enlarge": 30952, "\u0120twists": 30953, "\u0120faulty": 30954, "\u0120piracy": 30955, "\u0120impover": 30956, "\u0120rugged": 30957, "\u0120Fashion": 30958, "\u0120sands": 30959, "'?": 30960, "swick": 30961, "\u0120natives": 30962, "\u0120hen": 30963, "\u0120Noise": 30964, "\u00e3\u0125\u0139": 30965, "\u0120greens": 30966, "\u0120freezer": 30967, "\u0120dynasty": 30968, "\u0120Fathers": 30969, "\u0120Newark": 30970, "\u0120archaeological": 30971, "\u0120ot": 30972, "obar": 30973, "\u0120blockade": 30974, "\u0120allerg": 30975, "LV": 30976, "\u0120debit": 30977, "\u0120RFC": 30978, "\u0120Milton": 30979, "\u0120Pressure": 30980, "\u0120willingly": 30981, "\u0120disproportionate": 30982, "\u0120oppressive": 30983, "\u0120diamonds": 30984, "\u0120belongings": 30985, "1970": 30986, "\u0120bells": 30987, "\u0120imperialism": 30988, "\u0120227": 30989, "\u0120exploding": 30990, "\u0120Eclipse": 30991, "\u01201919": 30992, "\u0120rant": 30993, "\u0120nominations": 30994, "347": 30995, "\u0120peacefully": 30996, "rica": 30997, "\u0120FUCK": 30998, "\u0120vibration": 30999, "malink": 31000, "\u0120ropes": 31001, "\u0120Ivanka": 31002, "\u0120Brewery": 31003, "\u0120Booker": 31004, "\u0120Owens": 31005, "goers": 31006, "Services": 31007, "\u0120Snape": 31008, "\u0120191": 31009, "395": 31010, "\u0120299": 31011, "justice": 31012, "\u0120bri": 31013, "\u0120discs": 31014, "\u0120prominently": 31015, "\u0120vulgar": 31016, "\u0120skipping": 31017, "lves": 31018, "\u0120tsunami": 31019, "374": 31020, "\u0120Urug": 31021, "\u0120Eid": 31022, "recated": 31023, "phen": 31024, "\u0120faults": 31025, "\u0120Started": 31026, "950": 31027, "\u0120pi": 31028, "\u0120detector": 31029, "\u0120bastard": 31030, "\u0120validated": 31031, "SpaceEngineers": 31032, "OURCE": 31033, "\u0120(~": 31034, "\u0120unsur": 31035, "\u0120affirmed": 31036, "\u0120fascism": 31037, "\u0120resolving": 31038, "\u0120Chavez": 31039, "\u0120Cyn": 31040, "\u0120detract": 31041, "Lost": 31042, "\u0120rigged": 31043, "\u0120homage": 31044, "\u0120Bruno": 31045, "555": 31046, "eca": 31047, "\u0120presses": 31048, "\u0120humour": 31049, "\u0120spacing": 31050, "\u0120'/": 31051, "olkien": 31052, "Coun": 31053, "OPER": 31054, "Tre": 31055, "Son": 31056, "\u0120Cambodia": 31057, "ierre": 31058, "mong": 31059, "ozy": 31060, "\u0120liquidity": 31061, "\u0120Soviets": 31062, "\u0120Fernando": 31063, "\u0120229": 31064, "\u0120slug": 31065, "\u0120Catalan": 31066, "electric": 31067, "\u0120scenery": 31068, "\u0120Hearth": 31069, "\u0120constrained": 31070, "\u0120goalie": 31071, "\u0120Guidelines": 31072, "\u0120Ammo": 31073, "\u0120Pearson": 31074, "\u0120taxed": 31075, "\u0120fetus": 31076, "Response": 31077, "\u0120Alexis": 31078, "thia": 31079, "Guy": 31080, "\u0120reconstruct": 31081, "\u0120extremes": 31082, "\u0120concluding": 31083, "\u0120Peg": 31084, "ooks": 31085, "\u0120deductions": 31086, "Rose": 31087, "\u0120groundbreaking": 31088, "\u0120Targ": 31089, "\u00e3\u0125\u0123": 31090, "\u0120Reve": 31091, "resource": 31092, "\u0120moons": 31093, "\u0120electromagnetic": 31094, "\u0120amidst": 31095, "\u0120Viktor": 31096, "NESS": 31097, "BACK": 31098, "\u0120commute": 31099, "\u0120Anaheim": 31100, "\u0120fluctuations": 31101, "640": 31102, "\u0120noodles": 31103, "\u0120Copenhagen": 31104, "\u0120Tide": 31105, "\u0120Grizz": 31106, "\u0120SEE": 31107, "\u0120pipelines": 31108, "\u0120scars": 31109, "endo": 31110, "agus": 31111, "\u0120ETF": 31112, "/#": 31113, "\u0120Become": 31114, "448": 31115, "\u0120visc": 31116, "\u0120Recommended": 31117, "\u0120jumper": 31118, "\u0120cognition": 31119, "\u0120assassin": 31120, "\u0120witnessing": 31121, "\u0120Setup": 31122, "\u0120lac": 31123, "vim": 31124, "ISM": 31125, "pages": 31126, "SSL": 31127, "358": 31128, "\u0120adject": 31129, "industrial": 31130, "lore": 31131, "chery": 31132, "\u0120glitter": 31133, "\u0120calf": 31134, "Florida": 31135, "\u0120spoilers": 31136, "\u0120succeeds": 31137, "\u0120chanting": 31138, "\u0120slogans": 31139, "\u0120Tracy": 31140, "Visit": 31141, "rology": 31142, "\u0120mornings": 31143, "\u0120lineage": 31144, "\u0120sip": 31145, "\u0120intensely": 31146, "\u0120flourish": 31147, "\u0120Sleeping": 31148, "\u0120Fem": 31149, "orpor": 31150, "\u0120Klan": 31151, "\u0120Darth": 31152, "hack": 31153, "\u0120Nielsen": 31154, "\u0120tumors": 31155, "\u0120procurement": 31156, "\u0120Yorkshire": 31157, "\u0120raided": 31158, "KY": 31159, "Anna": 31160, "\u0120//[": 31161, "\u0120Disorder": 31162, "\u0120Mustang": 31163, "\u0120Wen": 31164, "\u0120Trying": 31165, "sq": 31166, "\u0120deliveries": 31167, "\u0120shutter": 31168, "\u0120cerebral": 31169, "\u0120bipolar": 31170, "\u0120CN": 31171, "lass": 31172, "jet": 31173, "\u0120debating": 31174, ">:": 31175, "\u0120eagle": 31176, "grades": 31177, "\u0120Dixon": 31178, "UGC": 31179, "MAS": 31180, "\u0120Draco": 31181, "\u0120Machines": 31182, "affer": 31183, "\u0120eman": 31184, "\u00c2\u00b2": 31185, "pron": 31186, "\u0120Gym": 31187, "\u0120comparatively": 31188, "\u0120Tribunal": 31189, "PRO": 31190, "\u0120lex": 31191, "\u0120fertile": 31192, "\u0120depressing": 31193, "\u0120superficial": 31194, "essential": 31195, "\u0120Hunters": 31196, "gp": 31197, "\u0120prominence": 31198, "Liber": 31199, "\u0120Ancest": 31200, "otechnology": 31201, "\u0120mocking": 31202, "\u0120Traff": 31203, "\u0138\u013c": 31204, "Medium": 31205, "Iraq": 31206, "\u0120psychiatrist": 31207, "Quantity": 31208, "\u0120Lect": 31209, "\u0120noisy": 31210, "520": 31211, "GY": 31212, "\u0120slapped": 31213, "\u0120MTV": 31214, "\u0120para": 31215, "pull": 31216, "Multiple": 31217, "asher": 31218, "\u0120nour": 31219, "\u0120Seg": 31220, "Spell": 31221, "vous": 31222, "ordial": 31223, "Senior": 31224, "\u0120Goldberg": 31225, "\u0120Plasma": 31226, "need": 31227, "\u0120messenger": 31228, "eret": 31229, "\u0120teamed": 31230, "\u0120literacy": 31231, "\u0120Leah": 31232, "\u0120Doyle": 31233, "\u0120emitted": 31234, "UX": 31235, "\u0120evade": 31236, "\u0120maze": 31237, "\u0120wrongly": 31238, "\u0120Lars": 31239, "\u0120stereotype": 31240, "\u0120pledges": 31241, "\u0120aroma": 31242, "\u0120MET": 31243, "\u0120acre": 31244, "\u0120OD": 31245, "\u0120ff": 31246, "\u0120breweries": 31247, "\u0120Hilton": 31248, "undle": 31249, "\u0120Kak": 31250, "\u0120Thankfully": 31251, "\u0120Canucks": 31252, "inctions": 31253, "\u0120Appears": 31254, "\u0120coer": 31255, "\u0120undermined": 31256, "rovers": 31257, "Andre": 31258, "\u0120blaze": 31259, "umers": 31260, "\u0120famine": 31261, "amphetamine": 31262, "ulkan": 31263, "Amount": 31264, "\u0120desperation": 31265, "wikipedia": 31266, "development": 31267, "\u0120Corinth": 31268, "ussia": 31269, "Jackson": 31270, "LI": 31271, "Native": 31272, "Rs": 31273, "Ohio": 31274, "\u0120Kathleen": 31275, "Fortunately": 31276, "\u0120attendant": 31277, "\u0120Preferred": 31278, "\u0120Didn": 31279, "\u0120Vs": 31280, "Mis": 31281, "\u0120respondent": 31282, "\u0120boun": 31283, "stable": 31284, "\u0120paved": 31285, "\u0120unexpl": 31286, "\u0120Cheney": 31287, "LM": 31288, "\u0120Cull": 31289, "blown": 31290, "\u0120confronting": 31291, "ocese": 31292, "serving": 31293, "Wi": 31294, "\u0120Lithuania": 31295, "anni": 31296, "\u0120stalk": 31297, "hd": 31298, "\u0120vener": 31299, "APH": 31300, "ynchronous": 31301, "URR": 31302, "umably": 31303, "historic": 31304, "Half": 31305, "Hay": 31306, "\u0120resilience": 31307, "spection": 31308, "\u0120abandoning": 31309, "Obs": 31310, "\u0120Debbie": 31311, "\u0120gradient": 31312, "\u0120Plaint": 31313, "\u0120Canal": 31314, "ARCH": 31315, "\u0120expansive": 31316, "\u0120fung": 31317, "\u0120bounced": 31318, "Und": 31319, "\u0120precautions": 31320, "\u0120clarification": 31321, "\u0120dagger": 31322, "\u0120grips": 31323, "\u0120\u00c2\u00b5": 31324, "\u0120Rivera": 31325, "\u0120Undead": 31326, "isites": 31327, "\u0120FIRST": 31328, "\u00c3\u00b1o": 31329, "audi": 31330, "\u0120hostages": 31331, "\u0120compliant": 31332, "\u0120alumni": 31333, "Seven": 31334, "\u0120cybersecurity": 31335, "either": 31336, "Collect": 31337, "\u0120invariably": 31338, "\u0120Soci": 31339, "\u0120lawmaker": 31340, "\u0120ale": 31341, "\u0120Personally": 31342, "Nazi": 31343, "\u0120customization": 31344, "\u0120Proc": 31345, "\u0120Saskatchewan": 31346, "eaturing": 31347, "\u0120spared": 31348, "\u0120discontinued": 31349, "\u0120computational": 31350, "\u0120Motorola": 31351, "\u0120supremacist": 31352, "governmental": 31353, "\u0120paradise": 31354, "\u0120Downing": 31355, "\u0120Nikon": 31356, "\u0120catalyst": 31357, "berra": 31358, "Toronto": 31359, "875": 31360, "beta": 31361, "\u0120Macron": 31362, "\u0120unrealistic": 31363, "vector": 31364, "\u0120Vehicles": 31365, "itiveness": 31366, "\u0120RV": 31367, "\u0120Colbert": 31368, "sin": 31369, "oji": 31370, "entin": 31371, "\u0120Krish": 31372, "hello": 31373, "ffield": 31374, "oky": 31375, "\u0120Tate": 31376, "\u0120maple": 31377, "\u0120aids": 31378, "chemical": 31379, "334": 31380, "nuts": 31381, "\u0120Warp": 31382, "\u0120xx": 31383, "\u0120Robb": 31384, "umerous": 31385, "_-_": 31386, "ftime": 31387, "\u0120VW": 31388, "\u0120winger": 31389, "\u0120Dome": 31390, "tools": 31391, "\u0120PV": 31392, "\u0120Georgetown": 31393, "\u0120geared": 31394, "\u0120jihadists": 31395, "\u0120cp": 31396, "\u0120steroids": 31397, "Mother": 31398, "clerosis": 31399, "\u0120DRM": 31400, "nesia": 31401, "\u0120linger": 31402, "\u0120immersive": 31403, "\u0120COUN": 31404, "\u0120outweigh": 31405, "ensual": 31406, "Band": 31407, "\u0120transforms": 31408, "matched": 31409, "psons": 31410, "\u0120Judicial": 31411, "factor": 31412, "\u0120referral": 31413, "\u0120oddly": 31414, "\u0120Wenger": 31415, "Bring": 31416, "\u0120Bows": 31417, "602": 31418, "ICLE": 31419, "\u0120lions": 31420, "\u0120Academic": 31421, "\u0120Thorn": 31422, "\u0120Raider": 31423, "kefeller": 31424, "Storage": 31425, "Lower": 31426, "\u0120Ort": 31427, "\u0120Equality": 31428, "ALT": 31429, "\u0120SOC": 31430, "Types": 31431, "\u0120lyn": 31432, "\u0120Asset": 31433, "coat": 31434, "TPP": 31435, "CVE": 31436, "\u0120Pioneer": 31437, "application": 31438, "Modern": 31439, "\u0120HK": 31440, "Environment": 31441, "Alright": 31442, "Rain": 31443, "IPP": 31444, "\u0120Shiite": 31445, "\u0120mound": 31446, "\u0120Abilities": 31447, "condition": 31448, "Staff": 31449, "\u0120competence": 31450, "\u0120Moor": 31451, "\u0120Diablo": 31452, "\u0120withheld": 31453, "\u0120ostensibly": 31454, "\u0120Brom": 31455, "\u0120msg": 31456, "\u0120denomin": 31457, "\u0120References": 31458, "\u0120FP": 31459, "\u0120plunged": 31460, "\u0120pamph": 31461, "moving": 31462, "central": 31463, "\u0120downright": 31464, "\u0120fading": 31465, "Tal": 31466, "Typ": 31467, "\u0120Thy": 31468, "ukes": 31469, "ithe": 31470, "\u0120ove": 31471, "\u0120battled": 31472, "\u0120seafood": 31473, "\u0120figur": 31474, "\u0120RD": 31475, "crop": 31476, "\u0120squads": 31477, "{\\": 31478, "\u00e0\u00b9": 31479, "\u0120Eh": 31480, "\u0120interviewing": 31481, "\u0120Qin": 31482, "\u0120aspiring": 31483, "PLIC": 31484, "\u0120clauses": 31485, "\u0120Gast": 31486, "\u0120Nir": 31487, "\u0120luggage": 31488, "\u0120hose": 31489, "\u0120systemd": 31490, "\u0120descending": 31491, "\u0120Revised": 31492, "\u0120Rails": 31493, "align": 31494, "709": 31495, "337": 31496, "\u0120fug": 31497, "charging": 31498, "tags": 31499, "\u0120uter": 31500, "kish": 31501, "WARNING": 31502, "490": 31503, "profits": 31504, "\u0120voyage": 31505, "\u0120ace": 31506, "\u0120Vanguard": 31507, "\u0120Tanks": 31508, "\u0120Muk": 31509, "\u0120226": 31510, "Safe": 31511, "Armor": 31512, "\u0120volcanic": 31513, "\u0120womb": 31514, "\u0120MIL": 31515, "\u0120beginner": 31516, "\u0120Recogn": 31517, "\u0120AAP": 31518, "PLAY": 31519, ")!": 31520, "\u0120detecting": 31521, "cn": 31522, "\u0120breaches": 31523, "Basically": 31524, "\u0120Pag": 31525, "\u0120Municipal": 31526, "\u0120Indie": 31527, "\u0120Laf": 31528, "\u0120Disable": 31529, "\u0120Olson": 31530, "\u0120restrained": 31531, "\u0120rulings": 31532, "\u0120humane": 31533, "events": 31534, "\u0120Cinema": 31535, "displayText": 31536, "\u0120Hatch": 31537, "actionDate": 31538, "onnaissance": 31539, "\u0120assaulting": 31540, "\u0120Lug": 31541, "CHAT": 31542, "\u0120vigorous": 31543, "\u0120Perse": 31544, "\u0120intolerance": 31545, "\u0120Snapchat": 31546, "\u0120Sharks": 31547, "\u0120dummy": 31548, "\u0120Diagn": 31549, "\u0120Guitar": 31550, "imeters": 31551, "403": 31552, "REG": 31553, "Ax": 31554, "\u0120separates": 31555, "\u0120Mahm": 31556, "\u0120tv": 31557, "jah": 31558, "OOL": 31559, "Circ": 31560, "\u0120Windsor": 31561, "ussian": 31562, "\u0120intuition": 31563, "\u0120disdain": 31564, "\u0120Donovan": 31565, "\u0120221": 31566, "Emb": 31567, "\u0120condemning": 31568, "\u0120generosity": 31569, "zzy": 31570, "\u0120panties": 31571, "\u0120Prevent": 31572, "ActionCode": 31573, "ANA": 31574, "342": 31575, "externalActionCode": 31576, "\u0120specifying": 31577, "\u0120crystall": 31578, "Jere": 31579, "\u0120rupt": 31580, "\u0120Apprentice": 31581, "\u0120profiling": 31582, "\u00d0\u00ba": 31583, "Strike": 31584, "\u0120sideline": 31585, "\u0120obligated": 31586, "\u0120occult": 31587, "\u0120bureaucratic": 31588, "antically": 31589, "rupted": 31590, "negative": 31591, "\u0120Ethiopia": 31592, "\u0120Civic": 31593, "\u0120insiders": 31594, "eligible": 31595, "\u0120TVs": 31596, "\u0120BAR": 31597, "\u0120TI": 31598, "iologist": 31599, "\u0120AIR": 31600, "\u0120substituted": 31601, "Arab": 31602, "\u0120Saul": 31603, "\u0120Yog": 31604, "prem": 31605, "\u0120builders": 31606, "\u0120stationary": 31607, "\u0120doubtful": 31608, "\u0120vigorously": 31609, "\u0120thrilling": 31610, "Physical": 31611, "\u0120Carey": 31612, "\u0120Hydra": 31613, "geoning": 31614, "\u0120Sly": 31615, "yton": 31616, "\u0120borrowers": 31617, "\u0120Parkinson": 31618, "\u0120\u00eb": 31619, "\u0120Jamaica": 31620, "\u0120satir": 31621, "\u0120insurgents": 31622, "\u0120Firm": 31623, "\u0120isot": 31624, "\u0120Karn": 31625, "ourning": 31626, "akens": 31627, "docs": 31628, "little": 31629, "\u0120Monaco": 31630, "CLASS": 31631, "Turkey": 31632, "Ly": 31633, "\u0120Conan": 31634, "assic": 31635, "\u0120starred": 31636, "\u0120Pacers": 31637, "eties": 31638, "\u0120tipping": 31639, "Moon": 31640, "\u0120Rw": 31641, "same": 31642, "\u0120cavity": 31643, "\u0120goof": 31644, "\u0120Zo": 31645, "Shock": 31646, "ummer": 31647, "\u0120emphasizes": 31648, "\u0120regrett": 31649, "\u0120novelty": 31650, "\u0120envy": 31651, "\u0120Passive": 31652, "rw": 31653, "505": 31654, "\u0120indifferent": 31655, "\u0120Rica": 31656, "\u0120Himself": 31657, "\u0120Freddie": 31658, "\u0120adip": 31659, "\u00e4\u00b8\u0122": 31660, "\u0120breakout": 31661, "\u0120hurried": 31662, "\u0120Huang": 31663, "\u0120Disk": 31664, "\u0120roaming": 31665, "?????-?????-": 31666, "UV": 31667, "\u0120Ricky": 31668, "\u0120Sigma": 31669, "\u0120marginalized": 31670, "\u0120edits": 31671, "\u0120304": 31672, "memory": 31673, "\u0120specimen": 31674, "293": 31675, "\u00e3\u0123\u00af": 31676, "\u0120vertically": 31677, "\u0120audition": 31678, "\u0120Heck": 31679, "\u0120caster": 31680, "\u0120Holdings": 31681, "adal": 31682, "\u0120Cron": 31683, "\u0120Liam": 31684, "\u0120deflect": 31685, "Pick": 31686, "\u0120Debug": 31687, "REF": 31688, "\u0120versatility": 31689, "othes": 31690, "classified": 31691, "\u0120Mahar": 31692, "\u0120Hort": 31693, "Counter": 31694, "stasy": 31695, "noticed": 31696, "331": 31697, "\u0120Shim": 31698, "fuck": 31699, "\u0120Bie": 31700, "\u0120airing": 31701, "\u0120Protein": 31702, "\u0120Holding": 31703, "\u0120spectators": 31704, "iliated": 31705, "\u0120Thatcher": 31706, "nosis": 31707, "\u00e3\u0125\u00bc\u00e3\u0125\u00b3": 31708, "Tele": 31709, "Boston": 31710, "\u0120Templ": 31711, "stay": 31712, "\u0120declarations": 31713, "479": 31714, "Volume": 31715, "\u0120Designer": 31716, "\u0120Overwatch": 31717, "idae": 31718, "\u0120onwards": 31719, "\u0120nets": 31720, "\u0120Manila": 31721, "particularly": 31722, "\u0120politic": 31723, "oother": 31724, "\u0120portraits": 31725, "\u0120pavement": 31726, "cffff": 31727, "\u0120saints": 31728, "\u0120beginners": 31729, "ESPN": 31730, "\u0120shortcomings": 31731, "\u00e2\u0137\u0132\u00e2\u0137\u0132": 31732, "\u0120comet": 31733, "\u0120Organic": 31734, "quel": 31735, "\u0120hospitalized": 31736, "Break": 31737, "\u0120peel": 31738, "dylib": 31739, "aspx": 31740, "urances": 31741, "\u0120TIM": 31742, "Pg": 31743, "\u0120readable": 31744, "\u0120Malik": 31745, "\u0120muzzle": 31746, "\u0120benchmarks": 31747, "dal": 31748, "\u0120Vacc": 31749, "\u0120Hicks": 31750, "609": 31751, "\u0120Biblical": 31752, "heng": 31753, "\u0120overload": 31754, "\u0120Civilization": 31755, "\u0120immoral": 31756, "\u0120fries": 31757, "\u00e3\u0124\u0134": 31758, "\u0120reproduced": 31759, "\u0120formulation": 31760, "jug": 31761, "irez": 31762, "gear": 31763, "\u0120coached": 31764, "MpServer": 31765, "\u0120SJ": 31766, "\u0120Kw": 31767, "Init": 31768, "deal": 31769, "\u0120Oro": 31770, "\u0120Loki": 31771, "\u0120Songs": 31772, "\u0120232": 31773, "\u0120Louise": 31774, "asionally": 31775, "\u0120uncond": 31776, "ollywood": 31777, "\u0120progressives": 31778, "\u0120Enough": 31779, "\u0120Doe": 31780, "\u0120wreckage": 31781, "\u0120brushed": 31782, "\u0120BaseType": 31783, "\u0120zoning": 31784, "ishable": 31785, "hetically": 31786, "\u0120Caucus": 31787, "\u0120Hue": 31788, "\u0120karma": 31789, "\u0120Sporting": 31790, "\u0120trader": 31791, "\u0120seeming": 31792, "\u0120Capture": 31793, "430": 31794, "bish": 31795, "\u0120tunes": 31796, "\u0120indoors": 31797, "\u0120Sphere": 31798, "\u0120Dancing": 31799, "TERN": 31800, "\u0120nob": 31801, "\u0120GST": 31802, "maps": 31803, "\u0120peppers": 31804, "Fit": 31805, "\u0120oversees": 31806, "\u0120Rabbi": 31807, "\u0120Ruler": 31808, "vertising": 31809, "office": 31810, "xxx": 31811, "\u0120raft": 31812, "Changed": 31813, "\u0120textbooks": 31814, "Links": 31815, "\u0120Omn": 31816, "\u00e3\u0122\u0133": 31817, "\u0120inconvenience": 31818, "\u0120Donetsk": 31819, "=~": 31820, "\u0120implicitly": 31821, "\u0120boosts": 31822, "\u0120Bones": 31823, "\u0120Boom": 31824, "Courtesy": 31825, "\u0120sensational": 31826, "ANY": 31827, "\u0120greedy": 31828, "eden": 31829, "\u0120inexper": 31830, "\u0120Ler": 31831, "\u0120Vale": 31832, "\u0120tighten": 31833, "\u0120EAR": 31834, "\u0120Num": 31835, "\u0120ancestor": 31836, "Sent": 31837, "\u0120Horde": 31838, "urgical": 31839, "allah": 31840, "\u0120sap": 31841, "amba": 31842, "\u0120Spread": 31843, "twitch": 31844, "\u0120grandson": 31845, "\u0120fracture": 31846, "\u0120moderator": 31847, "\u0120Seventh": 31848, "\u0120Reverse": 31849, "\u0120estimation": 31850, "Choose": 31851, "\u0120parach": 31852, "\u0120barric": 31853, "\u00e3\u0122\u0132": 31854, "\u0120compass": 31855, "\u0120allergic": 31856, "\u00e2\u0122\u0137": 31857, "OTHER": 31858, "errilla": 31859, "\u0120wagon": 31860, "\u0120zinc": 31861, "\u0120rubbed": 31862, "\u0120Fuller": 31863, "\u0120Luxembourg": 31864, "\u0120Hoover": 31865, "\u0120liar": 31866, "\u0120Evening": 31867, "\u0120Cobb": 31868, "esteem": 31869, "\u0120selector": 31870, "\u0120Brawl": 31871, "isance": 31872, "\u0120Ek": 31873, "\u0120troop": 31874, "\u0120guts": 31875, "\u0120Appeal": 31876, "\u0120Tibetan": 31877, "\u0120routines": 31878, "\u0120Ment": 31879, "\u0120summarized": 31880, "steamapps": 31881, "\u0120tranqu": 31882, "\u01201929": 31883, "oran": 31884, "\u0120Authent": 31885, "\u0120gmaxwell": 31886, "\u0120apprehens": 31887, "\u0120poems": 31888, "\u0120sausage": 31889, "\u0120Webster": 31890, "urus": 31891, "\u0120themed": 31892, "\u0120lounge": 31893, "\u0120charger": 31894, "Spoiler": 31895, "\u0120spilled": 31896, "hog": 31897, "\u0120Sunder": 31898, "\u0120Ain": 31899, "\u0120Angry": 31900, "\u0120disqual": 31901, "\u0120Frequency": 31902, "\u0120Ethernet": 31903, "\u0120helper": 31904, "Percent": 31905, "\u0120horrifying": 31906, "\u0120ail": 31907, "\u0120Allan": 31908, "EEE": 31909, "\u0120Crossing": 31910, "449": 31911, "\u0120holog": 31912, "\u0120Puzzles": 31913, "\u0120Goes": 31914, "erenn": 31915, "604": 31916, "\u00e3\u0123\u0131": 31917, "\u0120Rafael": 31918, "\u0120atten": 31919, "\u0120Emanuel": 31920, "\u0120upro": 31921, "\u0120Susp": 31922, "Psych": 31923, "\u0120Trainer": 31924, "\u0120NES": 31925, "\u0120Hunts": 31926, "becue": 31927, "\u0120counselor": 31928, "Rule": 31929, "\u0120toxins": 31930, "\u0120banners": 31931, "rifice": 31932, "\u0120greeting": 31933, "\u0120frenzy": 31934, "\u0120allocate": 31935, "\u0120*)": 31936, "expr": 31937, "503": 31938, "\u0120Chick": 31939, "\u0120Torn": 31940, "\u0120consolidation": 31941, "\u0120Fletcher": 31942, "switch": 31943, "frac": 31944, "clips": 31945, "\u0120McKin": 31946, "\u0120Lunar": 31947, "Month": 31948, "ITCH": 31949, "\u0120scholarly": 31950, "raped": 31951, "398": 31952, "\u01201910": 31953, "\u0120egreg": 31954, "\u0120insecure": 31955, "\u0120victorious": 31956, "cffffcc": 31957, "\u0120singled": 31958, "\u0120elves": 31959, "\u0120Wond": 31960, "burst": 31961, "\u0120camoufl": 31962, "\u0120BLACK": 31963, "\u0120conditioned": 31964, "\u00e7\u012b": 31965, "answered": 31966, "\u0120compulsory": 31967, "ascist": 31968, "\u0120podcasts": 31969, "\u0120Frankfurt": 31970, "bnb": 31971, "\u0120neoliberal": 31972, "\u0120Keyboard": 31973, "\u0120Belle": 31974, "warm": 31975, "\u0120trusts": 31976, "\u0120insured": 31977, "\u0120Bucc": 31978, "usable": 31979, "607": 31980, "\u0120Plains": 31981, "\u01201890": 31982, "\u0120sabotage": 31983, "\u0120lodged": 31984, "felt": 31985, "\u0120ga": 31986, "\u0120Narc": 31987, "\u0120Salem": 31988, "\u0120seventy": 31989, "\u0120Blank": 31990, "pocket": 31991, "\u0120whisper": 31992, "\u0120mating": 31993, "omics": 31994, "\u0120Salman": 31995, "\u0120Kad": 31996, "\u0120angered": 31997, "\u0120collisions": 31998, "\u0120extraordinarily": 31999, "\u0120coercion": 32000, "Ghost": 32001, "birds": 32002, "\u00e8\u0122": 32003, "kok": 32004, "\u0120permissible": 32005, "avorable": 32006, "\u0120pointers": 32007, "\u0120dissip": 32008, "aci": 32009, "\u0120theatrical": 32010, "\u0120Cosmic": 32011, "\u0120forgetting": 32012, "\u0120finalized": 32013, "\u00e5\u00a4\u00a7": 32014, "yout": 32015, "library": 32016, "\u0120booming": 32017, "\u0120Believe": 32018, "\u0120Teacher": 32019, "\u0120Liv": 32020, "\u0120GOODMAN": 32021, "\u0120Dominican": 32022, "ORED": 32023, "\u0120Parties": 32024, "\u0120precipitation": 32025, "\u0120Slot": 32026, "Roy": 32027, "\u0120Combined": 32028, "\u0120integrating": 32029, "\u0120chrome": 32030, "\u0120intestinal": 32031, "\u0120Rebell": 32032, "\u0120matchups": 32033, "\u0120blockbuster": 32034, "\u0120Loren": 32035, "\u0120Levy": 32036, "\u0120preaching": 32037, "\u0120Sending": 32038, "\u0120Purpose": 32039, "rax": 32040, "fif": 32041, "\u0120authoritative": 32042, "\u0120PET": 32043, "astical": 32044, "\u0120dishon": 32045, "\u0120chatting": 32046, "\u0120\"$:/": 32047, "Connection": 32048, "\u0120recreate": 32049, "\u0120delinqu": 32050, "\u0120broth": 32051, "\u0120Dirty": 32052, "\u0120Admin": 32053, "zman": 32054, "\u0120scholarships": 32055, "\u0120253": 32056, "contact": 32057, "alsa": 32058, "767": 32059, "creen": 32060, "abbage": 32061, "\u01201915": 32062, "\u0120blended": 32063, "\u0120alarmed": 32064, "Language": 32065, "356": 32066, "\u0120blends": 32067, "\u0120Changed": 32068, "Wolf": 32069, "\u0120hepat": 32070, "Creating": 32071, "\u0120persecut": 32072, "\u0120sweetness": 32073, "arte": 32074, "\u0120forfeiture": 32075, "\u0120Roberto": 32076, "impro": 32077, "NFL": 32078, "\u0120Magnet": 32079, "Detailed": 32080, "\u0120insignificant": 32081, "\u0120POLIT": 32082, "\u0120BBQ": 32083, "\u0120CPS": 32084, "\u0120seaw": 32085, "aminer": 32086, "mL": 32087, "endif": 32088, "finals": 32089, "\u0120265": 32090, "uish": 32091, "\u0120})": 32092, "\u0120Problems": 32093, "\u0120emblem": 32094, "\u0120seriousness": 32095, "\u0120parsing": 32096, "\u0120substitution": 32097, "\u0120pressured": 32098, "\u0120recycled": 32099, "aleb": 32100, "Ruby": 32101, "\u0120proficiency": 32102, "Driver": 32103, "\u0120Wester": 32104, ":'": 32105, "AFTA": 32106, "\u0120mantle": 32107, "\u0120Clayton": 32108, "flag": 32109, "\u0120practitioner": 32110, "covered": 32111, "\u0120Struct": 32112, "addafi": 32113, "425": 32114, "\u0120Township": 32115, "\u0120Hydro": 32116, "Louis": 32117, "343": 32118, "\u0120condo": 32119, "\u0120Tao": 32120, "\u0120utilization": 32121, "\u0120nausea": 32122, "\u0120Dems": 32123, "ridges": 32124, "pause": 32125, "\u0120formulas": 32126, "\u0120challenger": 32127, "376": 32128, "\u0120defective": 32129, "\u0120Railway": 32130, "\u0120PubMed": 32131, "\u0120yogurt": 32132, "lbs": 32133, "\u0120Norfolk": 32134, "OPE": 32135, "\u0120Moody": 32136, "\u0120distributor": 32137, "\u0120scrolls": 32138, "\u0120extracts": 32139, "Stan": 32140, "\u0120viability": 32141, "\u0120exposes": 32142, "\u0120starvation": 32143, "\u0120Steps": 32144, "\u0120Dodd": 32145, "few": 32146, "STD": 32147, "332": 32148, "\u0120closures": 32149, "\u0120complementary": 32150, "\u0120Sasha": 32151, "umpy": 32152, "\u0120monet": 32153, "\u0120articulate": 32154, "\u0120Doct": 32155, "killer": 32156, "\u0120scrim": 32157, "\u0120264": 32158, "\u0120prostitutes": 32159, "\u0120severed": 32160, "\u0120attachments": 32161, "\u0120cooled": 32162, "Lev": 32163, "\u0120Falk": 32164, "fail": 32165, "\u0120policeman": 32166, "\u0120Dag": 32167, "\u0120prayed": 32168, "\u0120Kernel": 32169, "\u0120clut": 32170, "\u0120cath": 32171, "\u0120anomaly": 32172, "Storm": 32173, "emaker": 32174, "\u0120Breakfast": 32175, "uli": 32176, "oire": 32177, "JJ": 32178, "hz": 32179, "Operation": 32180, "\u0120Sick": 32181, "354": 32182, "\u0120Guatemala": 32183, "Rate": 32184, "\u0120exposures": 32185, "faces": 32186, "\u0120Archae": 32187, "raf": 32188, "\u0120Mia": 32189, "\u01202025": 32190, "\u0120opaque": 32191, "\u0120disguised": 32192, "\u0120Headquarters": 32193, "Sah": 32194, "\u0120pots": 32195, "978": 32196, "\u0120Malf": 32197, "\u0120frowned": 32198, "\u0120poisonous": 32199, "\u0120Convers": 32200, "eeks": 32201, "\u0120crab": 32202, ".\"\"": 32203, "\u0120treason": 32204, "\u0120ranc": 32205, "\u0120escalating": 32206, "\u0120warr": 32207, "\u0120mobs": 32208, "\u0120lamps": 32209, "\u0120Sunshine": 32210, "\u0120Brunswick": 32211, "Phones": 32212, "\u0120spelled": 32213, "\u0120Skip": 32214, "\u01202050": 32215, "\u01201911": 32216, "\u0120Pluto": 32217, "\u0120Amend": 32218, "\u0120meats": 32219, "387": 32220, "\u0120stomp": 32221, "\u0120Zhou": 32222, "\u0120Leviathan": 32223, "\u0120Hazard": 32224, "adv": 32225, "\u0120Orwell": 32226, "\u0120aloud": 32227, "\u0120bumper": 32228, "\u0120Anarch": 32229, "ubuntu": 32230, "\u0120Serious": 32231, "fitting": 32232, "\u0120Optional": 32233, "\u0120Cecil": 32234, "REAM": 32235, "\u0120serotonin": 32236, "\u0120cultivate": 32237, "agogue": 32238, "}\\": 32239, "\u0120mosques": 32240, "\u0120Sunny": 32241, "\u0120reactive": 32242, "revolution": 32243, "\u0120Lup": 32244, "\u0120Fedora": 32245, "\u0120defenseman": 32246, "\u0120VID": 32247, "istine": 32248, "\u0120drowning": 32249, "\u0120Broadcasting": 32250, "\u0120thriller": 32251, "\u0120Scy": 32252, "\u0120accelerating": 32253, "\u0120directs": 32254, "odied": 32255, "bike": 32256, "duration": 32257, "\u0120painfully": 32258, "Redd": 32259, "\u0120productions": 32260, "\u0120gag": 32261, "\u0120whist": 32262, "\u0120sock": 32263, "\u0120infinitely": 32264, "\u0120Concern": 32265, "\u0120Citadel": 32266, "\u0120lieu": 32267, "\u0120candles": 32268, "ogeneous": 32269, "arger": 32270, "\u0120heavenly": 32271, "inflammatory": 32272, "Performance": 32273, "Cs": 32274, "ructose": 32275, "azaki": 32276, "\u0120pessim": 32277, "\u0120inference": 32278, "\u0120powd": 32279, "\u0120Zoe": 32280, "\u0120paints": 32281, "\u0120dazz": 32282, "pta": 32283, "-----------": 32284, "\u0120inspir": 32285, "\u0120Experimental": 32286, "\u0120Knife": 32287, "regor": 32288, "bors": 32289, "\u0120showers": 32290, "romeda": 32291, "\u0120saint": 32292, "\u0120benign": 32293, "\u0120Jiang": 32294, "\u0120envisioned": 32295, "\u0120shroud": 32296, "IFT": 32297, "HO": 32298, "\u0120shuff": 32299, "\u0120ICC": 32300, "\u0120segreg": 32301, "\u0120revisit": 32302, "ighthouse": 32303, "Li": 32304, "\u0120substrate": 32305, "\u0120Seas": 32306, "\u0120Reward": 32307, "\u0120Hep": 32308, "\u0120Brass": 32309, "sbm": 32310, "\u0120eliminates": 32311, "\u0120stamina": 32312, "\u0120VAT": 32313, "\u0120Loan": 32314, "\u0120constraint": 32315, "\u0120appropriated": 32316, "\u0120pes": 32317, "\u0120ALE": 32318, "ranging": 32319, "\u0120404": 32320, "392": 32321, "\u0120intellectuals": 32322, "achu": 32323, "\u0120restructuring": 32324, "\u0120Levin": 32325, "\u0120runes": 32326, "\u0120delightful": 32327, "\u0120carbohydrates": 32328, "\u0120Models": 32329, "\u0120Expo": 32330, "\u0120transporting": 32331, "alloc": 32332, "\u0120ringing": 32333, "Samsung": 32334, "\u0120scarcely": 32335, "\u0120URLs": 32336, "\u0120MAS": 32337, "\u0120prototypes": 32338, "\u0120narrator": 32339, "\u0120CPUs": 32340, "cdn": 32341, "\u0120Barton": 32342, "\u0120decidedly": 32343, "\u0120Shu": 32344, "ixir": 32345, "ocious": 32346, "\u0120Myst": 32347, "Nintendo": 32348, "\u0120reuse": 32349, "\u0120forgiven": 32350, "Few": 32351, "inical": 32352, "nat": 32353, "\u0120seamless": 32354, "\u0120Eva": 32355, "\u0120EVE": 32356, "\u0120JO": 32357, "landers": 32358, "\u0120softer": 32359, "negie": 32360, "\u0120transient": 32361, "\u0120orbital": 32362, "\u0120fulfil": 32363, "\u0120Kom": 32364, "Hopefully": 32365, "\u0120dynamically": 32366, "\u0120Hunger": 32367, "\u00e5\u013d": 32368, "\u0120Armenia": 32369, "elman": 32370, "berto": 32371, "\u0120pige": 32372, "\u0120IDs": 32373, "limit": 32374, "\u0120veins": 32375, "\u0120soaring": 32376, "packs": 32377, "Golden": 32378, "\u0120Crab": 32379, "istor": 32380, "\u0120RPM": 32381, "\u0120$$": 32382, "gression": 32383, "\u0120jihadist": 32384, "\u0120gamble": 32385, "\u0120careg": 32386, "\u0120inflated": 32387, "Face": 32388, "\u0120Firearms": 32389, "\u0120Emmanuel": 32390, "\u00e2\u013f": 32391, "\u0120shocks": 32392, "grab": 32393, "\u0120splend": 32394, "\u0120HPV": 32395, "abortion": 32396, "Above": 32397, "Entity": 32398, "players": 32399, "\u0120commenced": 32400, "ulence": 32401, "\u0120fulfillment": 32402, "\u0120embodiments": 32403, "\u0120Welfare": 32404, "\u0120hail": 32405, "\u0120<@": 32406, "tten": 32407, "\u0120catcher": 32408, "\u0120Jazeera": 32409, "\u0120volcano": 32410, "\u0120stabilize": 32411, "\u0120Handler": 32412, "\u0120intensified": 32413, "\u0120Abrams": 32414, "\u0120humiliation": 32415, "paced": 32416, "605": 32417, "\u0120CentOS": 32418, "Specific": 32419, "\u0120heed": 32420, "\u0120CAM": 32421, "\u0120Galile": 32422, "Die": 32423, "\u0120abolished": 32424, "\u0120Thomson": 32425, "\u0120Teachers": 32426, "\u0120Wass": 32427, "jong": 32428, "\u0120ISBN": 32429, "\u0120Allies": 32430, "shake": 32431, "\u00e5\u00b7": 32432, "vict": 32433, "Howard": 32434, "\u0120deem": 32435, "\u0120exceedingly": 32436, "\u0120Smartstocks": 32437, "ibe": 32438, "\u0120doorway": 32439, "\u0120competed": 32440, "igmat": 32441, "\u0120nationalists": 32442, "\u0120groom": 32443, "\u0120Keen": 32444, "\u0120disposable": 32445, "decl": 32446, "\u0120Tolkien": 32447, "\u0120Scheme": 32448, "\u0120biod": 32449, "\u0120avid": 32450, "\u0120Elon": 32451, "agar": 32452, "\u0120TSA": 32453, "Roman": 32454, "\u0120artificially": 32455, "\u0120advisors": 32456, "XL": 32457, "\u0120Inferno": 32458, "366": 32459, "\u0120tedious": 32460, "\u0120Photography": 32461, "\u0120Carrie": 32462, "\u0120trope": 32463, "\u0120Sandra": 32464, "\u0120decimal": 32465, "Queen": 32466, "\u0120Gundam": 32467, "\u0120OM": 32468, "otech": 32469, "NBA": 32470, "\u01201932": 32471, "\u0120entrenched": 32472, "\u0120Marion": 32473, "\u0120fraternity": 32474, "Labour": 32475, "Henry": 32476, "\u0120latitude": 32477, "Either": 32478, "\u0120enhances": 32479, "\u0120Potential": 32480, "\u0120shines": 32481, "idad": 32482, "\u0120breadth": 32483, "\u0120capacities": 32484, "\u0120\u00f0\u0141\u013b\u0124": 32485, "\u0120Bronx": 32486, "\u0120sexes": 32487, "\u0120differentiation": 32488, "\u0120heavyweight": 32489, "\u0120Taj": 32490, "dra": 32491, "\u0120migrate": 32492, "\u0120exhaustion": 32493, "\u0120RUN": 32494, "elsius": 32495, "\u0120Cuomo": 32496, "\u0120guitars": 32497, "\u0120clones": 32498, "\u0120Somew": 32499, "\u0120Pry": 32500, "-------------": 32501, "\u0120warranted": 32502, "cycles": 32503, "\u0120salvage": 32504, "\u0120disks": 32505, "RANT": 32506, "\u0120NGOs": 32507, "\u0120Martian": 32508, "\":[{\"": 32509, "\u0120addicts": 32510, "ojure": 32511, "illet": 32512, "\u0120amazingly": 32513, "artments": 32514, "pixel": 32515, "\u0120GPUs": 32516, "Layout": 32517, "\u00e8\u00a3": 32518, "\u0120Tamil": 32519, "\u0120Basil": 32520, "\u0120impartial": 32521, "\u0120Structure": 32522, "fork": 32523, "bryce": 32524, "\u0120ridge": 32525, "\u0120Hamburg": 32526, "rious": 32527, "\u0120blitz": 32528, "cigarettes": 32529, "\u0120canned": 32530, "402": 32531, "\u0120ironically": 32532, "\u0120compassionate": 32533, "\u0120Hawkins": 32534, ".#": 32535, "\u0120Cathedral": 32536, "\u0120rallied": 32537, "internal": 32538, "\u0120quota": 32539, "stakes": 32540, "TEXT": 32541, "mom": 32542, "\u0120completes": 32543, "\u0120238": 32544, "\u0120shrug": 32545, "\u00e3\u0125\u0133": 32546, "\u0120Ninth": 32547, "\u0120revise": 32548, "\u0120Provider": 32549, "\u0120treacher": 32550, "\u0120quasi": 32551, "\u0120PRES": 32552, "\u0120deposition": 32553, "\u0120confidentiality": 32554, "issors": 32555, "\u0120imbalance": 32556, "\u0120spanning": 32557, "\u0120angular": 32558, "\u0120Cul": 32559, "communication": 32560, "\u0120Nora": 32561, "\u0120Genius": 32562, "opter": 32563, "\u0120sacked": 32564, "Spot": 32565, "\u0120finely": 32566, "\u0120CHR": 32567, "282": 32568, "waves": 32569, "Palest": 32570, "\u0120Rohing": 32571, "NL": 32572, "\u00e8\u00bf": 32573, "\u0120shitty": 32574, "\u0120Scalia": 32575, "475": 32576, "Progress": 32577, "\u0120referencing": 32578, "\u0120classrooms": 32579, "abee": 32580, "\u0120sod": 32581, "hesion": 32582, "708": 32583, "\u0120Zuckerberg": 32584, "\u0120Finish": 32585, "\u0120Scotia": 32586, "\u0120Savior": 32587, "\u0120Installation": 32588, "antha": 32589, "(-": 32590, "\u0120302": 32591, "\u0120Punk": 32592, "\u0120crater": 32593, "youtu": 32594, "\u0120roast": 32595, "\u0120influencing": 32596, "\u0120dup": 32597, "\u0120JR": 32598, "\u0120Grav": 32599, "\u0120stature": 32600, "\u0120bathrooms": 32601, "Aside": 32602, "Wiki": 32603, "mean": 32604, "\u0120Zak": 32605, "\u0120Ones": 32606, "\u0120Nath": 32607, "\u0120hypert": 32608, "\u0120commencement": 32609, "Civil": 32610, "\u0120moderately": 32611, "\u0120distributors": 32612, "\u0120breastfeeding": 32613, "\u0120980": 32614, "\u0120Sik": 32615, "\u0120Cig": 32616, "\u0120AMER": 32617, "RIP": 32618, "\u0120Career": 32619, "usting": 32620, "\u0120messed": 32621, "\u0120eh": 32622, "\u0120Jensen": 32623, "/$": 32624, "\u0120blackmail": 32625, "\u0120conversions": 32626, "\u0120scientifically": 32627, "\u0120mantra": 32628, "paying": 32629, "\u0120ivory": 32630, "\u0120Courts": 32631, "OUGH": 32632, "auntlet": 32633, "Serial": 32634, "Brow": 32635, "\u0120Hundreds": 32636, "323": 32637, "\u0120pee": 32638, "\u0120linux": 32639, "\u0120submer": 32640, "\u0120Principal": 32641, "485": 32642, "\u0120DSL": 32643, "\u0120Cousins": 32644, "\u0120doctrines": 32645, "\u0120Athletics": 32646, "\u0120315": 32647, "\u0120Karma": 32648, "\u0120attent": 32649, "urger": 32650, "\u0120prescribe": 32651, "\u0120encaps": 32652, "\u0120Came": 32653, "\u0120secretive": 32654, "\u0120Crimes": 32655, "dn": 32656, "Clean": 32657, "\u0120Egyptians": 32658, "\u0120Carpenter": 32659, "\u0120ll": 32660, "Hum": 32661, "\u0120Milo": 32662, "\u0120capitalists": 32663, "\u0120briefed": 32664, "Twe": 32665, "\u0120Basin": 32666, "elvet": 32667, "Mos": 32668, "\u0120plunge": 32669, "\u0120Kaiser": 32670, "\u0120Fuj": 32671, "illin": 32672, "\u0120safeguards": 32673, "\u0120oste": 32674, "\u0120Opportunity": 32675, "\u0120Mafia": 32676, "\u0120Calling": 32677, "apa": 32678, "urban": 32679, "brush": 32680, "illard": 32681, "c\u00c3\u00a9": 32682, "intelligence": 32683, "\u0120Lob": 32684, "\u0120Druid": 32685, "\u0120smoother": 32686, "\u0120footing": 32687, "\u0120motorists": 32688, "arcity": 32689, "\u0120masculinity": 32690, "\u0120mism": 32691, "\u0120abdominal": 32692, "\u0120Tavern": 32693, "\u0120Roh": 32694, "\u0120escapes": 32695, "signed": 32696, "Anthony": 32697, "\u0120sacrificing": 32698, "\u0120intimacy": 32699, "\u0120anterior": 32700, "\u0120Kod": 32701, "\u0120motif": 32702, "\u0120graz": 32703, "\u0120visualization": 32704, "\u0120guitarist": 32705, "\u0120Trotsky": 32706, "magic": 32707, "Dar": 32708, "\u0120Mori": 32709, "\u0120wards": 32710, "\u0120toilets": 32711, "lest": 32712, "\u0120teleport": 32713, "\u0120Sundays": 32714, "\u0120Plat": 32715, "ETS": 32716, "\u0120eSports": 32717, "Patrick": 32718, "\u0120Katherine": 32719, "enko": 32720, "\u0120hassle": 32721, "\u0120Mick": 32722, "ggles": 32723, "\u0120hob": 32724, "aintain": 32725, "\u0120airborne": 32726, "\u0120spans": 32727, "\u0120chili": 32728, "\u0120aperture": 32729, "\u0120volunteered": 32730, "\u0120Incident": 32731, "\u0120Fres": 32732, "\u0120Veteran": 32733, "aughtered": 32734, "ingo": 32735, "\u0120uninsured": 32736, "CLOSE": 32737, "\u0120fuse": 32738, "\u0120erotic": 32739, "\u0120advertise": 32740, "raising": 32741, "Texture": 32742, "\u0120attends": 32743, "\u0120REAL": 32744, "uddled": 32745, "\u0120smoot": 32746, "\u0120305": 32747, "\u0120Willis": 32748, "\u0120blond": 32749, "Analysis": 32750, "\u0120VT": 32751, "onica": 32752, "\u0120stronghold": 32753, "RF": 32754, "NM": 32755, ".>>": 32756, "\u0120prosperous": 32757, "\u0120boasted": 32758, "292": 32759, "\u0120Manufacturing": 32760, "PRESS": 32761, "gren": 32762, "\u0120pharmacy": 32763, "\u0120Rockefeller": 32764, "kai": 32765, "\u0120thumbs": 32766, "\u0120Hut": 32767, "\u0120motherboard": 32768, "\u0120guardians": 32769, "\u0120Alter": 32770, "llular": 32771, "\u0120shack": 32772, "\u0120wisely": 32773, "\u0120backbone": 32774, "erva": 32775, "\u0120suicides": 32776, "\u0120McGregor": 32777, "ijah": 32778, "Emer": 32779, "\u0120Brav": 32780, "\u0120designate": 32781, "POST": 32782, "produced": 32783, "\u0120cleansing": 32784, "irlwind": 32785, "existent": 32786, "\u0120Humph": 32787, "\u0120Payne": 32788, "\u0120vested": 32789, "\u00c5\u00a1": 32790, "\u0120stringent": 32791, "iona": 32792, "\u0120unsub": 32793, "\u0120summed": 32794, "\u0120Hercules": 32795, "subject": 32796, "\u0120Ragnar": 32797, "\u0120Nos": 32798, "\u0120characterization": 32799, "\u0120savvy": 32800, "\u0120Dawson": 32801, "\u0120Casino": 32802, "\u0120fri": 32803, "\u0120Barrier": 32804, "\u0120misinformation": 32805, "\u0120insulation": 32806, "\u0120corridors": 32807, "\u0120airplanes": 32808, "\u0120Noct": 32809, "ahi": 32810, "\u01201916": 32811, "kb": 32812, "armac": 32813, "\u0120shun": 32814, "\u0120schema": 32815, "\u0120horrified": 32816, "\u0120239": 32817, "aunders": 32818, "NB": 32819, "iates": 32820, "erity": 32821, "\u0120Shard": 32822, "\u0120rarity": 32823, "\u0120grouped": 32824, "\u0120Ghana": 32825, "against": 32826, "\u0120Biological": 32827, "\u0120Aware": 32828, "owell": 32829, "\u00cf\u0126": 32830, "\u0120Beau": 32831, "shaw": 32832, "Hack": 32833, "\u0120Julius": 32834, "USS": 32835, "olson": 32836, "auna": 32837, "cru": 32838, "\u0120Maurice": 32839, "\u0120Ik": 32840, "\u0120sequencing": 32841, "\u0120radicals": 32842, "\u0120(?,": 32843, "virtual": 32844, "\u0120anyways": 32845, "\u0120reperc": 32846, "\u0120handlers": 32847, "\u0120hesitant": 32848, "\u00e9\u0125": 32849, "\u0120MF": 32850, "plementation": 32851, "associated": 32852, "\u0120campaigned": 32853, "\u0120Yue": 32854, "utations": 32855, "\u0120Yoga": 32856, "\u0120simmer": 32857, "\u0120rods": 32858, "\u0120melody": 32859, "\u0120convoy": 32860, "videos": 32861, "\u0120screened": 32862, "Neg": 32863, "ochemical": 32864, "\u0120())": 32865, "\u0120ultras": 32866, "\u0120antip": 32867, "\u0120Islanders": 32868, "704": 32869, "\u0120fetish": 32870, "\u0120ridiculously": 32871, "\u0120Kart": 32872, "\u0120mitochondrial": 32873, "\u0120interfering": 32874, "Builder": 32875, "\u0120overfl": 32876, "\u0120acne": 32877, "\u0120Mud": 32878, "\u0120Kerr": 32879, "flex": 32880, "\u0120Postal": 32881, "\u0120Baltic": 32882, "477": 32883, "\u0120Persons": 32884, "ourage": 32885, "HB": 32886, "\u0120Muse": 32887, "\u0120Immortal": 32888, "\u0120Driving": 32889, "\u0120petitions": 32890, "\u0120subscript": 32891, "\u0120sorce": 32892, "\u0120Processor": 32893, "uton": 32894, "Sony": 32895, "\u0120phon": 32896, "\u0120raced": 32897, "\u0120Anthrop": 32898, "\u0120daytime": 32899, "\u0120Exercise": 32900, "Adding": 32901, "\u0120engages": 32902, "\u0120Qualcomm": 32903, "\u0120miracles": 32904, "\u0120memes": 32905, "\u0120Drink": 32906, "\u0120Orioles": 32907, "\u0120hairs": 32908, "\u0120Polar": 32909, "athom": 32910, "\u0120slippery": 32911, "\u0120Remy": 32912, "\u0120caramel": 32913, "\u0120YEAR": 32914, "\u0120alk": 32915, "Ign": 32916, "aution": 32917, "\u0120Merlin": 32918, "\u0120Cran": 32919, "\u0120apologies": 32920, "\u0120410": 32921, "\u0120outing": 32922, "\u0120Memories": 32923, "appointed": 32924, "\u0120countered": 32925, "uld": 32926, "posing": 32927, "\u0120firewall": 32928, "\u0120Wast": 32929, "\u0120Wet": 32930, "worked": 32931, "seller": 32932, "\u0120repealed": 32933, "ereo": 32934, "assuming": 32935, "BLIC": 32936, "mite": 32937, "\u0120CEOs": 32938, "\u0120Chapel": 32939, "elligent": 32940, "________________________": 32941, "Dog": 32942, "\u0120wart": 32943, "\u0120subscriber": 32944, "sports": 32945, "\u0120begged": 32946, "\u0120MV": 32947, "\u0120semif": 32948, "ethical": 32949, "\u0120preach": 32950, "\u0120revital": 32951, "\u0120punitive": 32952, "\u0120shortcuts": 32953, "\u0120instituted": 32954, "\u0120Warsaw": 32955, "\u0120abdomen": 32956, "\u0120KING": 32957, "\u0120superintendent": 32958, "\u0120fry": 32959, "\u0120Geo": 32960, "TOR": 32961, "\u0120contradictions": 32962, "aptic": 32963, "\u0120landscapes": 32964, "bugs": 32965, "\u0120clust": 32966, "\u0120volley": 32967, "cribed": 32968, "\u0120tandem": 32969, "\u0120robes": 32970, "WHAT": 32971, "\u0120promoter": 32972, "\u0120eloqu": 32973, "reviewed": 32974, "\u0120DK": 32975, "\u0120Plato": 32976, "\u0120fps": 32977, "Tank": 32978, "\u0120Derrick": 32979, "\u0120prioritize": 32980, "asper": 32981, "\u0120Honduras": 32982, "\u0120Completed": 32983, "nec": 32984, "\u0120mog": 32985, "nir": 32986, "\u0120Mayo": 32987, "DEF": 32988, "stall": 32989, "inness": 32990, "\u0120Volkswagen": 32991, "\u0120precaution": 32992, "\u0120Mell": 32993, "iak": 32994, "istries": 32995, "\u0120248": 32996, "\u0120overlapping": 32997, "Senate": 32998, "\u0120Enhance": 32999, "resy": 33000, "racial": 33001, "ORTS": 33002, "\u0120Mormons": 33003, "Strong": 33004, "\u0120Coch": 33005, "Mexico": 33006, "\u0120Maduro": 33007, "\u0120jars": 33008, "\u0120cane": 33009, "Wik": 33010, "olla": 33011, "ifference": 33012, "\u0120physicist": 33013, "\u0120Maggie": 33014, "\u0120285": 33015, "\u0120depiction": 33016, "\u0120McLaren": 33017, "Ju": 33018, "\u0120slows": 33019, "\u0120commissioners": 33020, "\u0120Willow": 33021, "\u0120Explos": 33022, "hovah": 33023, "\u0120technician": 33024, "\u0120homicides": 33025, "\u0120Flav": 33026, "\u0120Truman": 33027, "\u012010000": 33028, "uctor": 33029, "\u0120shader": 33030, "Newsletter": 33031, "457": 33032, "\u0120rever": 33033, "\u0120hardened": 33034, "\u0120whereabouts": 33035, "\u0120redevelop": 33036, "\u0120carbs": 33037, "\u0120travers": 33038, "\u0120squirrel": 33039, "\u0120follower": 33040, "\u0120sings": 33041, "508": 33042, "\u0120rabbits": 33043, "emonium": 33044, "\u0120documenting": 33045, "\u0120misunderstood": 33046, ")'": 33047, "Rick": 33048, "ggies": 33049, "\u0120premie": 33050, "\u0120skating": 33051, "\u0120passports": 33052, "\u0120fists": 33053, "ageddon": 33054, "Haw": 33055, "ACP": 33056, "080": 33057, "\u0120Thoughts": 33058, "\u0120Carlson": 33059, "\u0120priesthood": 33060, "hua": 33061, "\u0120dungeons": 33062, "\u0120Loans": 33063, "\u0120antis": 33064, "\u0120familiarity": 33065, "\u0120Sabb": 33066, "opal": 33067, "\u0120Ink": 33068, "strike": 33069, "\u0120cram": 33070, "\u0120legalized": 33071, "\u0120cuisine": 33072, "\u0120fibre": 33073, "Travel": 33074, "\u0120Monument": 33075, "ODY": 33076, "ethy": 33077, "\u0120interstate": 33078, "\u0120PUR": 33079, "emporary": 33080, "\u0120Arabian": 33081, "developed": 33082, "\u0120saddle": 33083, "\u0120github": 33084, "\u0120Offer": 33085, "\u0120ISP": 33086, "rolet": 33087, "\u0120SUPER": 33088, "\u0120Denis": 33089, "\u0120multiplier": 33090, "\u0120stirred": 33091, "Interestingly": 33092, "\u0120customary": 33093, "\u0120billed": 33094, "hex": 33095, "\u0120multiplied": 33096, "\u0120flipping": 33097, "\u0120Crosby": 33098, "\u0120fundamentals": 33099, "iae": 33100, "\u0120Played": 33101, "\u0120Atom": 33102, "amazon": 33103, "\u0120Flam": 33104, "eez": 33105, "activated": 33106, "\u0120tablespoon": 33107, "\u0120liberalism": 33108, "\u0120Palin": 33109, "\u0120Patel": 33110, "Num": 33111, "\u0120TAM": 33112, "\u0120surn": 33113, "\u0120Reloaded": 33114, "\u0120coined": 33115, "\"],": 33116, "\u0120Clash": 33117, "\u0120Agu": 33118, "\u0120pragmatic": 33119, "\u0120Activate": 33120, "\u0120802": 33121, "\u0120trailers": 33122, "\u0120silhou": 33123, "\u0120probes": 33124, "\u0120circus": 33125, "\u0120Bain": 33126, "\u0120Lindsay": 33127, "\u0120Abbey": 33128, "Delivery": 33129, "\u0120concession": 33130, "\u0120gastro": 33131, "\u0120Sprite": 33132, "\u00c4\u0141": 33133, "andel": 33134, "\u0120gimm": 33135, "\u0120autobi": 33136, "\u0120Turtle": 33137, "\u0120wonderfully": 33138, "\u0120Haram": 33139, "\u0120Worldwide": 33140, "\u0120Handle": 33141, "\u0120theorists": 33142, "\u0120sleek": 33143, "\u0120Zhu": 33144, "ographically": 33145, "EGA": 33146, "\u0120Owners": 33147, "aths": 33148, "\u0120Antarctic": 33149, "natal": 33150, "=\"\"": 33151, "flags": 33152, "````": 33153, "\u0120sul": 33154, "Kh": 33155, "\u0120potassium": 33156, "\u0120lineman": 33157, "\u0120cereal": 33158, "\u0120Seasons": 33159, "\u01202022": 33160, "\u0120mathematic": 33161, "\u0120astronomers": 33162, "professional": 33163, "\u0120fares": 33164, "cknowled": 33165, "\u0120chi": 33166, "\u0120youngsters": 33167, "\u0120mistakenly": 33168, "\u0120hemisphere": 33169, "\u0120Divinity": 33170, "rone": 33171, "\u0120\",": 33172, "rings": 33173, "\u0120attracts": 33174, "vana": 33175, "\u00e5\u00b9": 33176, "CAP": 33177, "\u0120playlist": 33178, "\u0120porch": 33179, "\u00e3\u0123\u00a3": 33180, "\u0120incorporates": 33181, "\u0120soak": 33182, "\u0120asserting": 33183, "\u0120Terrorism": 33184, "\u0120Pablo": 33185, "Ja": 33186, "cester": 33187, "\u0120fearing": 33188, "\u0120Prayer": 33189, "\u0120escalated": 33190, "GW": 33191, "\u0120robe": 33192, "\u0120Brighton": 33193, "acists": 33194, "\u0120Symphony": 33195, "\u0120Dwarf": 33196, "\u0120Parade": 33197, "\u0120Lego": 33198, "\u0120inexpl": 33199, "\u0120lords": 33200, "leaf": 33201, "RAG": 33202, "liber": 33203, "\u0120cigars": 33204, "\u0120Jehovah": 33205, "606": 33206, "WINDOWS": 33207, "\u0120Liberia": 33208, "ebus": 33209, "Heavy": 33210, "\u0120lubric": 33211, "\u0120RW": 33212, "anguages": 33213, "\u0120narrowed": 33214, "computer": 33215, "\u0120Ember": 33216, "\u0120murdering": 33217, "\u0120downstream": 33218, "\u0120Tuls": 33219, "\u0120Tables": 33220, "Topic": 33221, "\u0120Accuracy": 33222, "=/": 33223, "lost": 33224, "\u0120Rei": 33225, "\u0120progresses": 33226, "bear": 33227, "\u0120establishments": 33228, "Justin": 33229, "\u0120Peach": 33230, "\u0120Gomez": 33231, "\u00e5\u00bf": 33232, "\u0120Triangle": 33233, "Ident": 33234, "\u0120Hive": 33235, "Resources": 33236, "\u0120mixes": 33237, "\u0120Assuming": 33238, "Mu": 33239, "\u0120hypoc": 33240, "\u0120sane": 33241, "\u0120Wan": 33242, "idious": 33243, "Success": 33244, "\u0120io": 33245, "Angel": 33246, "\u0120dangerously": 33247, "\u0120Creature": 33248, "WORK": 33249, ":[": 33250, "\u0120Katrina": 33251, "Listener": 33252, "Miller": 33253, "\u0120Idlib": 33254, "hang": 33255, "\u0120circumvent": 33256, "href": 33257, "\u0120celestial": 33258, "\u0120Weeks": 33259, "\u0120Pug": 33260, "\u0120Dalton": 33261, "\u0120subpoena": 33262, "uku": 33263, "\u0120persisted": 33264, "pei": 33265, "olding": 33266, "\u0120Documents": 33267, "\u0120Hast": 33268, "\u0120CENT": 33269, "\u0120primer": 33270, "\u0120synonymous": 33271, "\u0120nib": 33272, "ombs": 33273, "\u0120notation": 33274, "\u0120Dish": 33275, "\u0120Atmosp": 33276, "\u0120forbid": 33277, "\u0120ANG": 33278, "pattern": 33279, "los": 33280, "\u0120projectiles": 33281, "brown": 33282, ".\",": 33283, "\u0120Venom": 33284, "\u0120fiercely": 33285, "ublished": 33286, "\u0120Uran": 33287, "\u0120Nicarag": 33288, "410": 33289, "\u0120CAL": 33290, "OTOS": 33291, "\u0120Miracle": 33292, "\u0120Enchant": 33293, "\u0120guarding": 33294, "append": 33295, "Attach": 33296, "\u0120leveled": 33297, "\u0120condoms": 33298, "ihilation": 33299, "649": 33300, "\u0120nightmares": 33301, "\u0120THEY": 33302, "\u0120START": 33303, "\u0120Kinn": 33304, "\u0120roommate": 33305, "\u0120hygiene": 33306, "opping": 33307, "Job": 33308, "\u0120lvl": 33309, "\u0120VER": 33310, "\u0120Keeping": 33311, "abetic": 33312, "\u0120formatting": 33313, "erala": 33314, "\u0120revisions": 33315, "\u0120resurg": 33316, "Tel": 33317, "\u0120Goodman": 33318, "353": 33319, "pod": 33320, "\u0120indisp": 33321, "\u0120Translation": 33322, "\u0120gown": 33323, "\u0120Mund": 33324, "\u0120cis": 33325, "\u0120bystand": 33326, "collect": 33327, "\u0120Punjab": 33328, "actively": 33329, "\u0120Gamb": 33330, "tell": 33331, "\u0120importing": 33332, "gencies": 33333, "\u0120locom": 33334, "\u0120Brill": 33335, "Holy": 33336, "\u0120Berger": 33337, "\u0120showdown": 33338, "\u0120responders": 33339, "ILY": 33340, "\u0120takedown": 33341, "leted": 33342, "\u0120mattered": 33343, "\u0120predictive": 33344, "\u0120overlay": 33345, "GPU": 33346, "\u0120Vick": 33347, "\u0120conveyed": 33348, "Tab": 33349, "peer": 33350, "Scan": 33351, "\u0120defensively": 33352, "vae": 33353, "\u0120approving": 33354, "\u0120tiers": 33355, "\u0120Via": 33356, "querade": 33357, "\u0120Saudis": 33358, "\u0120demolished": 33359, "\u0120Prophe": 33360, "\u0120mono": 33361, "\u0120hospitality": 33362, "HAM": 33363, "\u0120Ariel": 33364, "MOD": 33365, "\u0120Torah": 33366, "\u0120blah": 33367, "\u0120Belarus": 33368, "erential": 33369, "\u0120Tuc": 33370, "\u0120banker": 33371, "397": 33372, "\u0120mosquit": 33373, "\u0120Scientist": 33374, "\u0120Musical": 33375, "\u0120hust": 33376, "Shift": 33377, "\u0120torment": 33378, "\u0120standoff": 33379, "Educ": 33380, "\u0120Fog": 33381, "\u0120amplifier": 33382, "Shape": 33383, "Instance": 33384, "\u0120Critics": 33385, "\u0120daemon": 33386, "Houston": 33387, "\u0120mattress": 33388, "\u0120IDF": 33389, "\u0120obscene": 33390, "\u0120Amer": 33391, "hetti": 33392, "\u0120compiling": 33393, "352": 33394, "verett": 33395, "\u0120Reduction": 33396, "istration": 33397, "\u0120Blessed": 33398, "\u0120Bachelor": 33399, "316": 33400, "\u0120prank": 33401, "\u0120Vulcan": 33402, "dding": 33403, "\u0120mourning": 33404, "\u0120Quint": 33405, "\u0120Blaster": 33406, "testing": 33407, "\u0120sediment": 33408, ">>>": 33409, "\u0120Eternity": 33410, "\u0120WHERE": 33411, "\u0120Maze": 33412, "\u0120reacting": 33413, "\u0120Alv": 33414, "omsday": 33415, "\u0120CRA": 33416, "\u0120translator": 33417, "\u0120bogus": 33418, "atu": 33419, "Website": 33420, "olls": 33421, "\u0120baptism": 33422, "\u0120sibling": 33423, "\u0120Autumn": 33424, "vez": 33425, "\u00e3\u0123\u00ae\u00e9": 33426, "guards": 33427, "Georg": 33428, "assadors": 33429, "\u0120Freud": 33430, "\u0120continents": 33431, "\u0120Registry": 33432, "Bernie": 33433, "\u0138\u013c\u00e5\u00a3\u00ab": 33434, "\u0120tolerant": 33435, "\u0120UW": 33436, "\u0120horribly": 33437, "995": 33438, "\u0120MIDI": 33439, "\u0120impatient": 33440, "ocado": 33441, "eri": 33442, "\u0120Worst": 33443, "\u0120Norris": 33444, "\u0120Talking": 33445, "\u0120defends": 33446, "ensable": 33447, "\u01202021": 33448, "\u0120anatomy": 33449, "Lew": 33450, "\u0120drawer": 33451, "\u0120Canberra": 33452, "\u0120patriotic": 33453, "\u00e9\u00be\u012f\u00e5\u0138\u013c\u00e5\u00a3\u00ab": 33454, "\u0120Avg": 33455, "ARM": 33456, "\u0120undisclosed": 33457, "\u0120farewell": 33458, "459": 33459, "bable": 33460, "\u0120Allison": 33461, "OLOG": 33462, "\u0120conco": 33463, "tight": 33464, "\u0120ACPI": 33465, "\u0120Mines": 33466, "lich": 33467, "\u0120\u00e2\u0136\u013e": 33468, "represented": 33469, "200000": 33470, "\u0120enthusiast": 33471, "OTS": 33472, "bil": 33473, "\u0120Ingredients": 33474, "\u0120inventor": 33475, "\u0120MySQL": 33476, "\u00c2\u0142\u00c2\u0142\u00c2\u0142": 33477, "\u0120ABOUT": 33478, "within": 33479, "\u0120mk": 33480, "Bul": 33481, "\u0120Fake": 33482, "\u0120draconian": 33483, "Wa": 33484, "helm": 33485, "\u0120Terran": 33486, "erville": 33487, "\u0120commonplace": 33488, "SIZE": 33489, "\u0120\"<": 33490, "replace": 33491, "ographs": 33492, "\u0120SELECT": 33493, "incible": 33494, "\u0120Mostly": 33495, "\u0120Sheffield": 33496, "\u0120IDE": 33497, "uggle": 33498, "\u0120citations": 33499, "hurst": 33500, "\u0120Unix": 33501, "\u0120unleash": 33502, "\u0120Piper": 33503, "\u0120Nano": 33504, "\u0120succumb": 33505, "\u0120reluctance": 33506, "\u01202500": 33507, "\u0120Merchant": 33508, "\u0120wiret": 33509, "\u0120combos": 33510, "\u0120Birthday": 33511, "\u0120charcoal": 33512, "\u0120UPS": 33513, "\u0120Fairfax": 33514, "\u0120driveway": 33515, "\u0120Tek": 33516, "\u0120Pitch": 33517, "overe": 33518, "\u0120technicians": 33519, "\u0120Actual": 33520, "flation": 33521, "\u0120Fiscal": 33522, "\u0120Empty": 33523, "anamo": 33524, "\u0120magnesium": 33525, "\u0120slut": 33526, "\u0120growers": 33527, "Investigators": 33528, "():": 33529, "\u0120Satellite": 33530, "\u0120Keynes": 33531, "missive": 33532, "lane": 33533, "\u0120borough": 33534, "344": 33535, "\u0120TEAM": 33536, "\u0120Bethesda": 33537, "CV": 33538, "hower": 33539, "\u0120RAD": 33540, "\u0120chant": 33541, "\u0120Riy": 33542, "\u0120compositions": 33543, "\u0120mildly": 33544, "\u0120meddling": 33545, "\u0120agility": 33546, "aneers": 33547, "501": 33548, "\u0120synth": 33549, "linger": 33550, "291": 33551, "\u0120exclaimed": 33552, "Party": 33553, "\u0120contamin": 33554, "\u0120Manor": 33555, "\u0120Respond": 33556, "\u0120praising": 33557, "\u0120manners": 33558, "fleet": 33559, "Summer": 33560, "\u0120Lynd": 33561, "\u0120Definitely": 33562, "grim": 33563, "\u0120bowling": 33564, "stri": 33565, "\u00e7\u013d": 33566, "ynt": 33567, "\u0120mandates": 33568, "DIV": 33569, "\u0120reconcile": 33570, "views": 33571, "\u0120Damon": 33572, "vette": 33573, "Flo": 33574, "\u0120Greatest": 33575, "ilon": 33576, "icia": 33577, "\u0120portrayal": 33578, "\u0120cushion": 33579, "504": 33580, "1979": 33581, "ossal": 33582, "Applic": 33583, "scription": 33584, "\u0120mitigation": 33585, "ATS": 33586, "pac": 33587, "\u0120erased": 33588, "\u0120deficiencies": 33589, "\u0120Hollande": 33590, "\u0120Xu": 33591, "\u0120bred": 33592, "\u0120pregnancies": 33593, "femin": 33594, "\u0120emph": 33595, "\u0120planners": 33596, "\u0120outper": 33597, "uttering": 33598, "\u0120perpetrator": 33599, "\u0120motto": 33600, "\u0120Ellison": 33601, "\u0120NEVER": 33602, "\u0120admittedly": 33603, "ARI": 33604, "\u0120Azerbaijan": 33605, "\u0120millisec": 33606, "\u0120combustion": 33607, "\u0120Bottle": 33608, "\u0120Lund": 33609, "\u0120Ps": 33610, "\u0120Dress": 33611, "\u0120fabricated": 33612, "\u0120battered": 33613, "\u0120sidel": 33614, "\u0120Notting": 33615, "Foreign": 33616, "\u0120Jerome": 33617, "020": 33618, "\u0120Arbit": 33619, "\u0120knots": 33620, "\u0120RIGHT": 33621, "Moving": 33622, "\u00e3\u0123\u013b": 33623, "\u0120surgeries": 33624, "\u0120courthouse": 33625, "\u0120mastered": 33626, "\u0120hovering": 33627, "\u0120Bran": 33628, "\u0120Alison": 33629, "\u0120safest": 33630, "military": 33631, "\u0120bullied": 33632, "\u0120barrage": 33633, "Reader": 33634, "ESE": 33635, "\u0120Geographic": 33636, "Tools": 33637, "314": 33638, "\u0120Geek": 33639, "roth": 33640, "glers": 33641, "\u0120FIN": 33642, "\u00cf\u0123": 33643, "\u0120Aston": 33644, "altern": 33645, "488": 33646, "\u0120veterin": 33647, "Gamer": 33648, "\u0120intel": 33649, "renches": 33650, "Shield": 33651, "\u0120amnesty": 33652, "\u0120Bhar": 33653, "\u0120piled": 33654, "\u0120honorable": 33655, "\u0120Institutes": 33656, "\u0120soaked": 33657, "\u0120coma": 33658, "\u0120EFF": 33659, "341": 33660, "bytes": 33661, "\u0120Gmail": 33662, "lein": 33663, "\u0120Canadiens": 33664, "material": 33665, "Il": 33666, "\u0120instructors": 33667, "\u0120KY": 33668, "\u0120conceive": 33669, "ubb": 33670, "\u0120Possible": 33671, "\u0120easing": 33672, "\u0120Christina": 33673, "\u0120caric": 33674, "\u0120HDR": 33675, "ROM": 33676, "\u0120shovel": 33677, "delete": 33678, "\u0120puff": 33679, "\u0120Changing": 33680, "\u0120seamlessly": 33681, "Attribute": 33682, "\u0120acquisitions": 33683, "akery": 33684, "\u0120EF": 33685, "\u0120autistic": 33686, "\u0120Takes": 33687, "\u0120Powder": 33688, "\u0120Stir": 33689, "510": 33690, "\u0120Bubble": 33691, "settings": 33692, "\u0120Fowler": 33693, "\u0120mustard": 33694, "\u0120moreover": 33695, "\u0120copyrighted": 33696, "\u0120LEDs": 33697, "1500": 33698, "\u00e6\u012b": 33699, "\u0120HIS": 33700, "enf": 33701, "\u0120custod": 33702, "\u0120Huck": 33703, "Gi": 33704, "\u0120img": 33705, "Answer": 33706, "Ct": 33707, "jay": 33708, "\u0120Infrastructure": 33709, "\u0120federally": 33710, "Loc": 33711, "\u0120microbes": 33712, "\u0120overrun": 33713, "dds": 33714, "otent": 33715, "adiator": 33716, ">>>>>>>>": 33717, "\u0120tornado": 33718, "\u0120adjud": 33719, "\u0120intrigued": 33720, "\u0120si": 33721, "\u0120Revelation": 33722, "progress": 33723, "\u0120burglary": 33724, "\u0120Saiyan": 33725, "\u0120Kathy": 33726, "\u0120serpent": 33727, "\u0120Andreas": 33728, "\u0120compel": 33729, "essler": 33730, "\u0120Plastic": 33731, "\u0120Advent": 33732, "\u0120Positive": 33733, "\u0120Qt": 33734, "\u0120Hindus": 33735, "registered": 33736, "ularity": 33737, "\u0120righteousness": 33738, "\u0120demonic": 33739, "uitive": 33740, "\u0120BDS": 33741, "\u0120Gregg": 33742, "cia": 33743, "\u0120Crusade": 33744, "\u0120Sinai": 33745, "WARE": 33746, "+(": 33747, "\u0120mell": 33748, "\u0120derail": 33749, "yards": 33750, "Ast": 33751, "\u0120noticeably": 33752, "\u0120Ober": 33753, "Ram": 33754, "\u0120unnoticed": 33755, "\u0120seq": 33756, "avage": 33757, "Ts": 33758, "\u0120640": 33759, "\u0120concede": 33760, "\u0120])": 33761, "Fill": 33762, "\u0120captivity": 33763, "\u0120Improvement": 33764, "\u0120Crusader": 33765, "araoh": 33766, "MAP": 33767, "\u00e6\u0139": 33768, "\u0120stride": 33769, "always": 33770, "Fly": 33771, "Nit": 33772, "\u0120algae": 33773, "\u0120Cooking": 33774, "\u0120Doors": 33775, "Malley": 33776, "\u0120policemen": 33777, "\u00e3\u0123\u012f": 33778, "\u0120astronaut": 33779, "accessible": 33780, "495": 33781, "\u0120RAW": 33782, "cliffe": 33783, "udicrous": 33784, "\u0120depended": 33785, "alach": 33786, "\u0120ventures": 33787, "rake": 33788, "\u0120tits": 33789, "\u0120Hou": 33790, "\u0120condom": 33791, "ormonal": 33792, "\u0120indent": 33793, "\u0120uploading": 33794, "Footnote": 33795, "Important": 33796, "\u0120271": 33797, "\u0120mindful": 33798, "\u0120contends": 33799, "Cra": 33800, "\u0120calibr": 33801, "\u0120OECD": 33802, "plugin": 33803, "Fat": 33804, "\u0120ISS": 33805, "\u0120Dynamics": 33806, "ansen": 33807, "686": 33808, "'),": 33809, "\u0120sprite": 33810, "\u0120handheld": 33811, "\u0120Hipp": 33812, "=~=~": 33813, "Trust": 33814, "\u0120semantics": 33815, "\u0120Bundes": 33816, "\u0120Reno": 33817, "\u0120Literature": 33818, "sense": 33819, "Gary": 33820, "\u0120Aeg": 33821, "\u0120Trin": 33822, "EEK": 33823, "\u0120cleric": 33824, "\u0120SSH": 33825, "\u0120christ": 33826, "\u0120invading": 33827, "ibu": 33828, "\u0120enum": 33829, "aura": 33830, "\u0120allege": 33831, "\u0120Incredible": 33832, "BBC": 33833, "\u0120thru": 33834, "\u0120sailed": 33835, "\u0120emulate": 33836, "\u0120insecurity": 33837, "\u0120crou": 33838, "\u0120accommodations": 33839, "\u0120incompetent": 33840, "\u0120slips": 33841, "\u0120Earthqu": 33842, "sama": 33843, "ILLE": 33844, "\u0120iPhones": 33845, "asaki": 33846, "\u0120bye": 33847, "\u0120ard": 33848, "\u0120extras": 33849, "\u0120slaughtered": 33850, "\u0120crowdfunding": 33851, "resso": 33852, "\u0120filib": 33853, "\u0120ERROR": 33854, "\u0120TLS": 33855, "egg": 33856, "\u0120Ital": 33857, "\u0120enlist": 33858, "\u0120Catalonia": 33859, "\u0120Scots": 33860, "\u0120sergeant": 33861, "\u0120dissolve": 33862, "NH": 33863, "\u0120standings": 33864, "rique": 33865, "IQ": 33866, "\u0120beneficiary": 33867, "\u0120aquarium": 33868, "YouTube": 33869, "\u0120PowerShell": 33870, "\u0120brightest": 33871, "\u0120Warrant": 33872, "Sold": 33873, "Writing": 33874, "\u0120beginnings": 33875, "\u0120Reserved": 33876, "\u0120Latinos": 33877, "heading": 33878, "\u0120440": 33879, "\u0120rooftop": 33880, "ATING": 33881, "\u0120390": 33882, "VPN": 33883, "Gs": 33884, "kernel": 33885, "turned": 33886, "\u0120preferable": 33887, "\u0120turnovers": 33888, "\u0120Hels": 33889, "Sa": 33890, "\u0120Shinji": 33891, "veh": 33892, "\u0120MODULE": 33893, "Viol": 33894, "\u0120exiting": 33895, "\u0120jab": 33896, "\u0120Vanilla": 33897, "\u0120acron": 33898, "\u0120Gap": 33899, "bern": 33900, "Ak": 33901, "\u0120McGu": 33902, "\u0120endlessly": 33903, "\u0120Farage": 33904, "\u0120Noel": 33905, "Va": 33906, "MK": 33907, "\u0120brute": 33908, "\u0120Kru": 33909, "\u0120ESV": 33910, "\u0120Olivia": 33911, "\u00e2\u0122\u0142": 33912, "\u0120Kaf": 33913, "\u0120trusting": 33914, "\u0120hots": 33915, "324": 33916, "\u0120malaria": 33917, "\u0120json": 33918, "\u0120pounding": 33919, "ortment": 33920, "Country": 33921, "\u0120postponed": 33922, "\u0120unequiv": 33923, "?),": 33924, "\u0120Rooney": 33925, "udding": 33926, "\u0120Leap": 33927, "urrence": 33928, "shapeshifter": 33929, "\u0120HAS": 33930, "osate": 33931, "\u0120cavern": 33932, "\u0120conservatism": 33933, "\u0120BAD": 33934, "\u0120mileage": 33935, "\u0120arresting": 33936, "Vaults": 33937, "\u0120mixer": 33938, "Democratic": 33939, "\u0120Benson": 33940, "\u0120authored": 33941, "8000": 33942, "\u0120proactive": 33943, "\u0120Spiritual": 33944, "tre": 33945, "\u0120incarcerated": 33946, "\u0120Sort": 33947, "\u0120peaked": 33948, "\u0120wielding": 33949, "reciation": 33950, "\u00d7\u013b\u00d7": 33951, "Patch": 33952, "\u0120Emmy": 33953, "\u0120exqu": 33954, "tto": 33955, "\u0120Ratio": 33956, "\u0120Picks": 33957, "\u0120Gry": 33958, "phant": 33959, "\u0120fret": 33960, "\u0120ethn": 33961, "\u0120archived": 33962, "%-": 33963, "cases": 33964, "\u0120Blaze": 33965, "\u0120imb": 33966, "cv": 33967, "yss": 33968, "imony": 33969, "\u0120countdown": 33970, "\u0120awakening": 33971, "\u0120Tunisia": 33972, "\u0120Refer": 33973, "\u0120MJ": 33974, "\u0120unnatural": 33975, "\u0120Carnegie": 33976, "izen": 33977, "\u0120Nuggets": 33978, "hess": 33979, "\u0120evils": 33980, "647": 33981, "\u0120introductory": 33982, "loving": 33983, "\u0120McMahon": 33984, "\u0120ambiguity": 33985, "Label": 33986, "\u0120Almighty": 33987, "\u0120coloring": 33988, "\u0120Claus": 33989, "setting": 33990, "NULL": 33991, "\u0120Favorite": 33992, "\u0120SIG": 33993, ">(": 33994, "\u0120Shiva": 33995, "\u0120Mayer": 33996, "\u0120stormed": 33997, "\u0120Coverage": 33998, "weapons": 33999, "igham": 34000, "\u0120unanswered": 34001, "\u0120leve": 34002, "\u0120coy": 34003, "cas": 34004, "bags": 34005, "asured": 34006, "Seattle": 34007, "\u0120Santorum": 34008, "serious": 34009, "\u0120courageous": 34010, "\u0120Soup": 34011, "\u0120confiscated": 34012, "\u0120///": 34013, "\u0120unconventional": 34014, "\u0120moms": 34015, "\u0120Rohingya": 34016, "\u0120Orchestra": 34017, "\u0120Potion": 34018, "\u0120discredit": 34019, "\u0120FIL": 34020, "fixed": 34021, "\u0120Deer": 34022, "doi": 34023, "\u0120Dimension": 34024, "\u0120bureaucrats": 34025, "eteen": 34026, "\u0120actionGroup": 34027, "ohm": 34028, "\u0120bumps": 34029, "\u0120Utility": 34030, "\u0120submarines": 34031, "renheit": 34032, "research": 34033, "\u0120Shapiro": 34034, "\u0120sketches": 34035, "\u0120deceptive": 34036, "\u0120Vil": 34037, "esame": 34038, "\u0120Essentially": 34039, "\u0120rampage": 34040, "isky": 34041, "\u0120muttered": 34042, "thritis": 34043, "\u0120236": 34044, "fet": 34045, "bars": 34046, "\u0120pupil": 34047, "\u0120Thou": 34048, "oS": 34049, "song": 34050, "\u0120fractured": 34051, "\u0120revert": 34052, "picture": 34053, "\u0120criterion": 34054, "usher": 34055, "\u0120repercussions": 34056, "\u0120Vintage": 34057, "\u0120Superintendent": 34058, "Officers": 34059, "\u0120flagged": 34060, "\u0120blames": 34061, "\u0120inverse": 34062, "ographers": 34063, "\u0120makeshift": 34064, "\u0120devoid": 34065, "\u0120fossils": 34066, "\u0120Aristotle": 34067, "\u0120Funds": 34068, "\u0120depleted": 34069, "\u0120Flu": 34070, "\u0120Yuan": 34071, "\u0120woes": 34072, "\u0120lipid": 34073, "\u0120situ": 34074, "requisites": 34075, "\u0120furnish": 34076, "\u0120Samar": 34077, "\u0120shameful": 34078, "\u0120adversely": 34079, "\u0120adept": 34080, "\u0120remorse": 34081, "\u0120murderous": 34082, "uckles": 34083, "\u0120ESL": 34084, "\u0120314": 34085, "sent": 34086, "\u0120redef": 34087, "\u0120Cache": 34088, "\u0120Purs": 34089, "igans": 34090, "\u0120460": 34091, "\u0120prescriptions": 34092, "\u0120fres": 34093, "Fuck": 34094, "ocrates": 34095, "Twenty": 34096, "\u0120Weird": 34097, "\u0120Toggle": 34098, "\u0120Called": 34099, "itizens": 34100, "\u0120poultry": 34101, "\u0120harvesting": 34102, "\u00e3\u0124\u00a6\u00e3\u0124\u00b9": 34103, "Bottom": 34104, "\u0120cautioned": 34105, "tn": 34106, "396": 34107, "\u0120Nikki": 34108, "\u0120evaluations": 34109, "\u0120harassing": 34110, "\u0120bindings": 34111, "\u0120Monetary": 34112, "\u0120hitters": 34113, "\u0120adversary": 34114, "unts": 34115, "\u0120setback": 34116, "\u0120encrypt": 34117, "\u0120Cait": 34118, "\u0120lows": 34119, "enges": 34120, "\u0120Norn": 34121, "\u0120bulbs": 34122, "\u0120bottled": 34123, "\u0120Voyager": 34124, "317": 34125, "\u0120spheres": 34126, "politics": 34127, "\u0120subtract": 34128, "\u0120sensations": 34129, "\u0120appalling": 34130, "\u0120316": 34131, "\u0120environmentally": 34132, "\u0120STEM": 34133, "\u0120publishes": 34134, "560": 34135, "\u0120diligence": 34136, "484": 34137, "\u0120advises": 34138, "\u0120petrol": 34139, "\u0120imagining": 34140, "\u0120patrols": 34141, "\u0120Integer": 34142, "\u0120Ashes": 34143, "actus": 34144, "\u0120Radiant": 34145, "\u0120LT": 34146, "itability": 34147, "htaking": 34148, "Setting": 34149, "\u0120nuanced": 34150, "\u0120Reef": 34151, "\u0120Developers": 34152, "Ni": 34153, "pieces": 34154, "990": 34155, "License": 34156, "\u0120lowers": 34157, "\u0120Ottoman": 34158, "327": 34159, "ooo": 34160, "\u0120quitting": 34161, "markets": 34162, "Behind": 34163, "\u0120basin": 34164, "\u0120docs": 34165, "anie": 34166, "flash": 34167, "ctl": 34168, "\u0120civilized": 34169, "\u0120Fukushima": 34170, "\"],\"": 34171, "\u0120KS": 34172, "\u0120Honestly": 34173, "arat": 34174, "\u0120constructs": 34175, "\u0120Lans": 34176, "\u0120Dire": 34177, "\u0120LIKE": 34178, "\u0120Trouble": 34179, "\u0120withholding": 34180, "\u0120Oblivion": 34181, "\u0120sanity": 34182, "anya": 34183, "Const": 34184, "\u0120grocer": 34185, "\u0120Celsius": 34186, "\u0120recounted": 34187, "\u0120Wife": 34188, "Border": 34189, "atered": 34190, "happy": 34191, "\u0120spoiler": 34192, "\u0120logically": 34193, "Hall": 34194, "\u0120succeeding": 34195, "\u0120polymorph": 34196, "\u0120axes": 34197, "\u0120Shotgun": 34198, "\u0120Slim": 34199, "\u0120Principles": 34200, "\u0120Leth": 34201, "arta": 34202, "\u0120scor": 34203, "Screenshot": 34204, "\u0120relaxation": 34205, "#$#$": 34206, "\u0120deterrent": 34207, "iddy": 34208, "\u0120powerless": 34209, "\u0120lesbians": 34210, "\u0120chords": 34211, "\u0120Edited": 34212, "selected": 34213, "\u0120separatists": 34214, "0002": 34215, "\u0120airspace": 34216, "\u0120turnaround": 34217, "\u0120cunning": 34218, "PATH": 34219, "Poly": 34220, "\u0120bombed": 34221, "\u0120tion": 34222, "xs": 34223, "\u0120withhold": 34224, "\u0120waged": 34225, "\u0120Liberties": 34226, "Flag": 34227, "\u0120comforting": 34228, "454": 34229, "\u0120Iris": 34230, "arers": 34231, "\u0120rag": 34232, "\u0120relocated": 34233, "\u0120Guarant": 34234, "\u0120strategically": 34235, "\u0120gamma": 34236, "uberty": 34237, "\u0120Lockheed": 34238, "gres": 34239, "\u0120grilled": 34240, "\u0120Lowe": 34241, "stats": 34242, "\u0120Rocks": 34243, "\u0120sensing": 34244, "\u0120renting": 34245, "\u0120Geological": 34246, "\u00d8\u00a7\u00d8": 34247, "otrop": 34248, "\u0120sew": 34249, "\u0120improperly": 34250, "486": 34251, "\u0120\u00e2\u0138\u0142": 34252, "\u0120starving": 34253, "\u0120Bj": 34254, "Discussion": 34255, "328": 34256, "\u0120Combo": 34257, "\u0120Fixes": 34258, "NAT": 34259, "\u0120striving": 34260, "thora": 34261, "\u0120harvested": 34262, "\u0120Ping": 34263, "\u0120playful": 34264, "\u0120avenues": 34265, "\u0120occupational": 34266, "\u0120wakes": 34267, "\u0120Courier": 34268, "\u0120drummer": 34269, "\u0120Browser": 34270, "\u0120Houth": 34271, "itu": 34272, "\u0120apparel": 34273, "paste": 34274, "\u0120hunted": 34275, "\u0120Secondly": 34276, "lain": 34277, "XY": 34278, "\u0120PIN": 34279, "icons": 34280, "\u0120cocktails": 34281, "\u0120sizable": 34282, "\u0120hurdles": 34283, "estinal": 34284, "\u0120Recreation": 34285, "\u0120eco": 34286, "648": 34287, "\u0120Died": 34288, "mint": 34289, "\u0120fingerprints": 34290, "\u0120dispose": 34291, "\u0120Bosnia": 34292, "tsy": 34293, "2200": 34294, "\u0120inspected": 34295, "\u0120Fou": 34296, "\u0120fuss": 34297, "\u0120ambush": 34298, "\u0120Rak": 34299, "\u0120manifested": 34300, "Prosecut": 34301, "\u0120suffice": 34302, "rences": 34303, "\u0120compensated": 34304, "\u0120Cyrus": 34305, "\u0120genus": 34306, "\u0120Wolverine": 34307, "\u0120Trends": 34308, "\u0120hikes": 34309, "\u0120Seen": 34310, "\u0120enrol": 34311, "Cold": 34312, "\u0120politely": 34313, "\u0120Slav": 34314, "\u0120Rupert": 34315, "\u0120eyewitness": 34316, "\u0120Alto": 34317, "\u0120uncomp": 34318, "\u0120posterior": 34319, "Must": 34320, "\u0120Herz": 34321, "\u0120progressively": 34322, "\u0120234": 34323, "\u0120indifference": 34324, "\u0120Cunningham": 34325, "\u0120academia": 34326, "\u0120sewer": 34327, "\u0120astounding": 34328, "\u0120AES": 34329, "rather": 34330, "\u0120eldest": 34331, "\u0120climbs": 34332, "\u0120Adds": 34333, "\u0120outcry": 34334, "\u0120contag": 34335, "\u0120Houses": 34336, "\u0120pept": 34337, "\u0120Melania": 34338, "interested": 34339, "\u0120UCH": 34340, "\u0120Roots": 34341, "\u0120Hubbard": 34342, "\u0120TBD": 34343, "\u0120Romanian": 34344, "filename": 34345, "Stone": 34346, "\u0120Impl": 34347, "\u0120chromosome": 34348, "Cle": 34349, "dx": 34350, "\u0120scrambled": 34351, "\u0120Pt": 34352, "\u0120242": 34353, "OPLE": 34354, "\u0120tremendously": 34355, "Street": 34356, "\u0120craving": 34357, "\u0120bundled": 34358, "\u0120RG": 34359, "pipe": 34360, "\u0120injuring": 34361, "\u0120arcane": 34362, "Particip": 34363, "\u0120Heroic": 34364, "sty": 34365, "\u0120topping": 34366, "\u0120Tempest": 34367, "rentices": 34368, "bh": 34369, "\u0120paranoia": 34370, "\u0120Unicode": 34371, "\u0120egregious": 34372, "\u0120\\'": 34373, "\u0120Oswald": 34374, "\u0120gravel": 34375, "\u0120Simpsons": 34376, "\u0120bland": 34377, "\u0120Guantanamo": 34378, "Writer": 34379, "liners": 34380, "\u0120Dice": 34381, "JC": 34382, "\u0120parity": 34383, "\u0120sided": 34384, "\u0120237": 34385, "\u0120Pyrrha": 34386, "atters": 34387, "dk": 34388, "Fine": 34389, "compan": 34390, "\u0120formulated": 34391, "\u0120Idol": 34392, "ilers": 34393, "hemoth": 34394, "\u0120Fav": 34395, "\u0120intrusion": 34396, "\u0120carrots": 34397, "\u0120Layer": 34398, "\u0120Hacker": 34399, "\u0120----------------": 34400, "\u0120moderation": 34401, "\u00e9\u0123": 34402, "ococ": 34403, "\u0120characterize": 34404, "\u0120Teresa": 34405, "\u0120socioeconomic": 34406, "\u0120perk": 34407, "\u0120Participation": 34408, "training": 34409, "\u0120Paulo": 34410, "phys": 34411, "\u0120trustworthy": 34412, "\u0120embodied": 34413, "\u0120Merch": 34414, "currency": 34415, "\u0120Priority": 34416, "\u0120teasing": 34417, "\u0120absorbing": 34418, "\u0120unfinished": 34419, "\u0120Comparison": 34420, "\u0120disple": 34421, "writers": 34422, "\u0120professions": 34423, "\u0120Penguin": 34424, "\u0120angrily": 34425, "\u0120LINK": 34426, "688": 34427, "\u0120Correspond": 34428, "\u0120prevailed": 34429, "\u0120cartel": 34430, "lp": 34431, "asms": 34432, "\u0120Redemption": 34433, "\u0120Islamists": 34434, "effects": 34435, "dose": 34436, "\u0120Latter": 34437, "\u0120Halifax": 34438, "\u0120vas": 34439, "\u0120Topics": 34440, "\u0120Named": 34441, "advertising": 34442, "zza": 34443, "ICES": 34444, "\u0120retarded": 34445, "achable": 34446, "\u0120Puppet": 34447, "\u0120ItemLevel": 34448, "\u0120retract": 34449, "\u0120identifiable": 34450, "Aaron": 34451, "\u0120Buster": 34452, "sol": 34453, "helle": 34454, "assemb": 34455, "Hope": 34456, "ranged": 34457, "Ba": 34458, "\u0120Purch": 34459, "\u00e9\u0122": 34460, "\u0120Siri": 34461, "\u0120arrivals": 34462, "\u01201912": 34463, "\u0120shortened": 34464, "\u0120312": 34465, "\u0120discrepancy": 34466, "\u0120Temperature": 34467, "\u0120Walton": 34468, "\u0120kinderg": 34469, "polit": 34470, "\u0120remix": 34471, "\u0120connectors": 34472, "\u00e3\u0125\u013a\u00e3\u0125\u00a9": 34473, "\u0120Kazakhstan": 34474, "dominated": 34475, "\u0120sugars": 34476, "imble": 34477, "\u0120Panic": 34478, "\u0120Demand": 34479, "\u0120Colony": 34480, "onen": 34481, "\u0120MER": 34482, "775": 34483, "uria": 34484, "azaar": 34485, "\u0120Degree": 34486, "Pri": 34487, "\u0120sunshine": 34488, "\u0120251": 34489, "\u0120psychedelic": 34490, "\u0120digitally": 34491, "\u0120Braun": 34492, "\u0120shimmer": 34493, "\u0120shave": 34494, "\u0120Telesc": 34495, "\u0120Astral": 34496, "\u0120Venezuelan": 34497, "\u0120OG": 34498, "\u0120crawling": 34499, "Integ": 34500, "\u0120Feather": 34501, "\u0120unfolding": 34502, "\u0120appropriation": 34503, "\u0120\u00e8\u00a3\u0131\u00e8": 34504, "\u0120Mobility": 34505, "\u0120Ney": 34506, "-.": 34507, "bilt": 34508, "LIN": 34509, "\u0120Tube": 34510, "\u0120Conversely": 34511, "\u0120keyboards": 34512, "\u0120Cao": 34513, "\u0120overth": 34514, "\u0120laure": 34515, ">>\\": 34516, "\u0120Viper": 34517, "acha": 34518, "Offset": 34519, "\u0120Raleigh": 34520, "\u0120Jae": 34521, "Jordan": 34522, "jp": 34523, "\u0120totalitarian": 34524, "Connector": 34525, "\u0120observes": 34526, "\u0120Spartan": 34527, "\u0120Immediately": 34528, "\u0120Scal": 34529, "Cool": 34530, "\u0120taps": 34531, "\u0120roar": 34532, "Past": 34533, "\u0120chars": 34534, "\u0120Bender": 34535, "\u0120Sheldon": 34536, "\u0120painter": 34537, "\u0120beacon": 34538, "\u0120Creatures": 34539, "\u0120downturn": 34540, "\u0120hinder": 34541, "\u0120Andromeda": 34542, "\u00c3\u013d": 34543, "ccoli": 34544, "\u0120Fitness": 34545, "etrical": 34546, "\u0120utilizes": 34547, "\u0120senate": 34548, "\u0120ensemble": 34549, "\u0120cheers": 34550, "TW": 34551, "\u0120affluent": 34552, "kil": 34553, "rylic": 34554, "ordering": 34555, "Computer": 34556, "\u0120gruesome": 34557, "ostics": 34558, "\u0120Ubisoft": 34559, "\u0120Kelley": 34560, "\u0120wrench": 34561, "\u0120bourgeoisie": 34562, "IBLE": 34563, "\u0120Preston": 34564, "worn": 34565, "arist": 34566, "reating": 34567, "\u0120stained": 34568, "arine": 34569, "\u0120slime": 34570, "ENN": 34571, "\u0120chests": 34572, "\u0120groundwater": 34573, "annot": 34574, "\u0120Tray": 34575, "\u0120Locke": 34576, "\u0120CTR": 34577, "\u0120dudes": 34578, "\u0120External": 34579, "\u0120Decoder": 34580, "\u0120paramed": 34581, "\u0120Medline": 34582, "809": 34583, "\u0120Dinner": 34584, "rupal": 34585, "gz": 34586, "\u0120Gum": 34587, "\u0120Demo": 34588, "jee": 34589, "\u0120dh": 34590, "berman": 34591, "archs": 34592, "\u0120enqu": 34593, "\u0120Epstein": 34594, "\u0120devastation": 34595, "\u0120friendships": 34596, "\u0120Ard": 34597, "\u0120231": 34598, "\u0120Rubin": 34599, "\u0120Distance": 34600, "\u0120spurred": 34601, "\u0120dossier": 34602, "\u0120overlooking": 34603, "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\": 34604, "Forest": 34605, "\u0120Comes": 34606, "\\\",": 34607, "\u0120Iranians": 34608, "\u0120fixtures": 34609, "Laughs": 34610, "\u0120curry": 34611, "\u0120Kingston": 34612, "\u0120squash": 34613, "\u0120catalogue": 34614, "\u0120abnormalities": 34615, "\u0120digestive": 34616, ".........": 34617, "\u0120subordinate": 34618, "ogly": 34619, "\u0120249": 34620, "Middle": 34621, "\u0120massac": 34622, "\u0120burgers": 34623, "\u0120downstairs": 34624, "\u01201931": 34625, "394": 34626, "\u0120VG": 34627, "\u0120lasers": 34628, "\u0120Sikh": 34629, "\u0120Alexa": 34630, "derived": 34631, "\u0120cyclist": 34632, "\u00e3\u0123\u00ae\u00e9\u0143\u0136": 34633, "oneliness": 34634, "!!!!!!!!": 34635, "\u0120buffs": 34636, "legate": 34637, "\u0120raping": 34638, "\u0120recommending": 34639, "rored": 34640, "\u0120multicultural": 34641, "unique": 34642, "\u0120businessmen": 34643, "\u0120uneasy": 34644, "\u0120MAP": 34645, "\u0120dispersed": 34646, "cipline": 34647, "Jess": 34648, "\u0120Kerala": 34649, "\u00e5\u00a7": 34650, "\u0120abstraction": 34651, "Surv": 34652, "Uh": 34653, "\u0120printers": 34654, "ija": 34655, "owder": 34656, "\u0120analogous": 34657, "\u0120ASP": 34658, "afer": 34659, "\u0120unfolded": 34660, "\u0120leveling": 34661, "\u0120breached": 34662, "\u0120Hearing": 34663, "\u0120nat": 34664, "\u0120translating": 34665, "critical": 34666, "\u0120antagonist": 34667, "\u0120Yesterday": 34668, "\u0120fuzzy": 34669, "wash": 34670, "mere": 34671, "\u0120bewild": 34672, "\u0120Mae": 34673, "Virgin": 34674, "phrase": 34675, "\u0120signaled": 34676, "\u0120HIGH": 34677, "\u0120protester": 34678, "\u0120garner": 34679, "unknown": 34680, "\u0120kay": 34681, "\u0120abducted": 34682, "\u0120stalking": 34683, "amn": 34684, "\u0120deserving": 34685, "\u0120Riv": 34686, "\u0120Jorge": 34687, "\u0120scratching": 34688, "\u0120Saving": 34689, "iping": 34690, "\u0120tease": 34691, "\u0120missionary": 34692, "\u0120Morrow": 34693, "TIME": 34694, "Present": 34695, "\u0120chemotherapy": 34696, "terness": 34697, "\u0120Homes": 34698, "\u0120Purdue": 34699, "\u0120staunch": 34700, "\u0120Whitney": 34701, "\u0120THERE": 34702, "\u00ce\u00bc": 34703, "iatus": 34704, "\u0120Ernest": 34705, "\u0120Deploy": 34706, "\u0120coveted": 34707, "FML": 34708, "\u0120Dialogue": 34709, "\u0120exited": 34710, "fruit": 34711, "\u0120nerd": 34712, "\":\"\",\"": 34713, "\u0120vivo": 34714, "ruly": 34715, "460": 34716, "\u0120Amen": 34717, "rehensible": 34718, "\u0120\u00e2\u013a": 34719, "DIR": 34720, "\u0120adherence": 34721, "\u0120chew": 34722, "\u0120Coke": 34723, "\u0120Sergei": 34724, "digital": 34725, "\u0120Neck": 34726, "gently": 34727, "enthal": 34728, "/)": 34729, "\u0120weary": 34730, "\u0120guise": 34731, "\u0120Concord": 34732, "\u0120Onion": 34733, "atcher": 34734, "\u0120binge": 34735, "\u0120Directive": 34736, "\u0120manned": 34737, "ansk": 34738, "\u0120illusions": 34739, "\u0120billionaires": 34740, "383": 34741, "olyn": 34742, "odynamic": 34743, "\u0120Wheat": 34744, "\u0120Alic": 34745, "\u0120coloured": 34746, "\u0120NAFTA": 34747, "abo": 34748, "\u0120macros": 34749, "independent": 34750, "sweet": 34751, "\u0120spac": 34752, "\u0120Kabul": 34753, "\u0120\u00c4": 34754, "eme": 34755, "\u0120dictated": 34756, "\u0120shouts": 34757, "={": 34758, "\u0120ripping": 34759, "\u0120Shay": 34760, "\u0120Cricket": 34761, "directed": 34762, "\u0120analysed": 34763, "\u0120WARRANT": 34764, "agons": 34765, "\u0120Blazers": 34766, "\u0120cheered": 34767, "\u0120arithmetic": 34768, "\u0120Tanz": 34769, "373": 34770, "\u0120Flags": 34771, "\u0120295": 34772, "\u0120witches": 34773, "\u0120Included": 34774, "\u0120Gained": 34775, "\u0120Blades": 34776, "Gam": 34777, "\u0120Samantha": 34778, "\u0120Atlantis": 34779, "\u0120Pratt": 34780, "\u0120spoiled": 34781, "\u0120IB": 34782, "\u0120Ramirez": 34783, "Probably": 34784, "rero": 34785, "\u0120Ng": 34786, "\u0120Warlock": 34787, "tp": 34788, "\u0120overhe": 34789, "\u0120administrations": 34790, "\u0120tint": 34791, "\u0120regiment": 34792, "\u0120pistols": 34793, "\u0120blankets": 34794, "\u0120epist": 34795, "\u0120bowls": 34796, "\u0120hydraulic": 34797, "\u0120dean": 34798, "\u0120jung": 34799, "\u0120ascend": 34800, "705": 34801, "\u0120Santiago": 34802, "\u00c3\u00ae": 34803, "\u0120unavoid": 34804, "\u0120Shaman": 34805, "reb": 34806, "\u0120stemming": 34807, "998": 34808, "\u0120MG": 34809, "sticks": 34810, "esthesia": 34811, "ERO": 34812, "\u0120morbid": 34813, "\u0120Grill": 34814, "\u0120Poe": 34815, "anyl": 34816, "\u0120deleting": 34817, "\u0120Surveillance": 34818, "\u0120directives": 34819, "\u0120iterations": 34820, "\u0120Rox": 34821, "\u0120Milky": 34822, "Father": 34823, "\u0120patented": 34824, "447": 34825, "\u0120precursor": 34826, "\u0120maiden": 34827, "\u0120Phen": 34828, "\u0120Vegan": 34829, "\u0120Patent": 34830, "Kelly": 34831, "Redditor": 34832, "\u0120nods": 34833, "\u0120ventilation": 34834, "\u0120Schwarz": 34835, "\u0120wizards": 34836, "\u0120ominous": 34837, "\u0120Heads": 34838, "\u0120BG": 34839, "\u0120lumber": 34840, "\u0120Spiel": 34841, "\u0120isEnabled": 34842, "\u0120ancestral": 34843, "\u0120Ships": 34844, "\u0120wrestler": 34845, "phi": 34846, "\u0120yuan": 34847, "\u0120Rebellion": 34848, "\u0120iceberg": 34849, "\u0120magically": 34850, "\u0120diversion": 34851, "arro": 34852, "ythm": 34853, "\u0120Riders": 34854, "\u0120Robbie": 34855, "\u0120Kara": 34856, "\u0120Maintenance": 34857, "\u0120Herb": 34858, "\u0120harms": 34859, "packed": 34860, "\u0120Feinstein": 34861, "\u0120marrying": 34862, "\u0120blending": 34863, "\u0120Rates": 34864, "\u01201880": 34865, "\u0120wrink": 34866, "\u0120Unch": 34867, "\u0120Torch": 34868, "described": 34869, "\u0120humanoid": 34870, "ilitating": 34871, "\u0120Conv": 34872, "\u0120Feld": 34873, "IGHTS": 34874, "\u0120whistleblower": 34875, "ortmund": 34876, "etsy": 34877, "arrett": 34878, "\u0120Mono": 34879, "\u0120Ike": 34880, "\u0120CNBC": 34881, "\u0120WAY": 34882, "\u0120MDMA": 34883, "\u0120Individuals": 34884, "\u0120supplemental": 34885, "\u0120powerhouse": 34886, "\u0120Stru": 34887, "Focus": 34888, "aphael": 34889, "\u0120Colleg": 34890, "atti": 34891, "ZA": 34892, "\u0120perenn": 34893, "\u0120Signature": 34894, "\u0120Rodney": 34895, "\u0120cubes": 34896, "iddled": 34897, "\u0120Dante": 34898, "\u0120INV": 34899, "ilingual": 34900, "\u0120Cth": 34901, "\u0120sofa": 34902, "\u0120intimidate": 34903, "\u0120Roe": 34904, "\u0120Diplom": 34905, "\u0120Countries": 34906, "ayson": 34907, "\u0120extradition": 34908, "\u0120disabling": 34909, "\u0120Cardiff": 34910, "\u0120memorandum": 34911, "\u0120Trace": 34912, "\u0120???": 34913, "sector": 34914, "\u0120Rouhani": 34915, "\u0120Yates": 34916, "\u0120Freeze": 34917, "\u0120bladder": 34918, "Motor": 34919, "\u0120Promise": 34920, "antasy": 34921, "\u0120foreseeable": 34922, "\u0120Cologne": 34923, "container": 34924, "\u0120Trees": 34925, "\u0120Gors": 34926, "\u0120Sinclair": 34927, "\u0120barring": 34928, "keye": 34929, "\u0120slashed": 34930, "\u0120Statistical": 34931, "\u00e9\u0129": 34932, "\u0120\u00e2\u0138\u00ba": 34933, "Allows": 34934, "\u0120humility": 34935, "\u0120drilled": 34936, "\u0120Furn": 34937, "443": 34938, "\u0120sewage": 34939, "\u0120homepage": 34940, "\u0120courtyard": 34941, "\u0120vile": 34942, "\u0120subsidiaries": 34943, "ajo": 34944, "directory": 34945, "\u0120ammon": 34946, "Vers": 34947, "charges": 34948, "\u0120}}": 34949, "\u0120Chains": 34950, "\u0120246": 34951, "nob": 34952, "\u0120percept": 34953, "\u0120grit": 34954, "\u0120fishermen": 34955, "\u0120Iraqis": 34956, "\u0120DISTR": 34957, "\u0120FULL": 34958, "\u0120Evaluation": 34959, "graph": 34960, "atial": 34961, "\u0120cooperating": 34962, "\u0120melan": 34963, "\u0120enlightened": 34964, "\u0120ali": 34965, "tailed": 34966, "\u0120salute": 34967, "\u0120weakest": 34968, "\u0120Bulldogs": 34969, "UA": 34970, "\u0120Alloy": 34971, "\u0120semen": 34972, "ocene": 34973, "\u0120Williamson": 34974, "spr": 34975, ",\u00e2\u0122\u0136": 34976, "\u0120GF": 34977, "ittens": 34978, "Beat": 34979, "\u0120Junk": 34980, "iphate": 34981, "\u0120Farmers": 34982, "\u0120Bitcoins": 34983, "igers": 34984, "dh": 34985, "\u0120Loyal": 34986, "payer": 34987, "\u0120entertained": 34988, "\u0120penned": 34989, "\u0120coupon": 34990, "Queue": 34991, "\u0120weakening": 34992, "carry": 34993, "\u0120underestimate": 34994, "\u0120shootout": 34995, "\u0120charismatic": 34996, "\u0120Procedure": 34997, "\u0120prudent": 34998, "inances": 34999, "\u0120riches": 35000, "\u0120cortical": 35001, "\u0120strides": 35002, "\u0120drib": 35003, "\u0120Oilers": 35004, "540": 35005, "\u0120Perform": 35006, "\u0120Bangkok": 35007, "\u0120euth": 35008, "SER": 35009, "\u0120simplistic": 35010, "tops": 35011, "campaign": 35012, "Quality": 35013, "\u0120impoverished": 35014, "\u0120Eisenhower": 35015, "\u0120augment": 35016, "\u0120Harden": 35017, "\u0120intervened": 35018, "\u0120listens": 35019, "\u0120Kok": 35020, "\u0120sage": 35021, "\u0120rubbish": 35022, "\u0120Ded": 35023, "\u0120mull": 35024, "pelling": 35025, "\u0120videot": 35026, "Production": 35027, "DJ": 35028, "miah": 35029, "\u0120adaptations": 35030, "\u0120medically": 35031, "\u0120boarded": 35032, "\u0120arrogance": 35033, "\u0120scrapped": 35034, "\u0120oppress": 35035, "FORMATION": 35036, "\u0120junction": 35037, "415": 35038, "EEEE": 35039, "Skill": 35040, "\u0120subdu": 35041, "\u0120Suggest": 35042, "\u0120Pett": 35043, "\u0120lett": 35044, "\u0120Manip": 35045, "\u0120Caf": 35046, "\u0120Cooperation": 35047, "Ther": 35048, "\u0120regained": 35049, "\u00b6\u00e6": 35050, "reflect": 35051, "\u0120thugs": 35052, "\u0120Shelby": 35053, "\u0120dictates": 35054, "\u0120Weiner": 35055, "\u0120Hale": 35056, "\u0120battleground": 35057, "schild": 35058, "\u0120condol": 35059, "hunt": 35060, "ositories": 35061, "\u0120accuses": 35062, "Filename": 35063, "\u0120shri": 35064, "\u0120motivate": 35065, "\u0120reflections": 35066, "Null": 35067, "\u0120Lobby": 35068, "\u00a5\u00b5": 35069, "\u0120SATA": 35070, "\u0120Backup": 35071, "\u00d1\u0125": 35072, "nin": 35073, "\u0120Correction": 35074, "\u0120juicy": 35075, "utra": 35076, "\u0120Pric": 35077, "\u0120restraining": 35078, "\u0120Airbnb": 35079, "\u0120Arrest": 35080, "\u0120appropriations": 35081, "\u0120slopes": 35082, "\u0120manslaughter": 35083, "\u0120workings": 35084, "\u0120Huss": 35085, "\u0120Frey": 35086, "Leave": 35087, "\u0120Harmony": 35088, "\u0120Feder": 35089, "\u0120430": 35090, "\u0120trench": 35091, "\u0120gladly": 35092, "\u0120bullpen": 35093, "\u0120Gau": 35094, "bones": 35095, "\u0120groove": 35096, "\u0120pretext": 35097, "\u00e3\u0127\u012d": 35098, "\u0120transmitter": 35099, "\u0120Component": 35100, "\u0120underage": 35101, "\u0120Empires": 35102, "Tile": 35103, "\u0120oy": 35104, "\u0120Marvin": 35105, "\u0120CAS": 35106, "\u0120bloss": 35107, "\u0120replicated": 35108, "\u0120Mariners": 35109, "Marcus": 35110, "\u0120Blocks": 35111, "\u0120liberated": 35112, "\u0120butterfly": 35113, "Feel": 35114, "\u0120fermentation": 35115, "\u0120youtube": 35116, "\u0120offend": 35117, "\u0120Term": 35118, "resist": 35119, "\u0120cessation": 35120, "\u0120insurgency": 35121, "\u0120bir": 35122, "\u0120Raise": 35123, "595": 35124, "\u0120hypotheses": 35125, "502": 35126, "\u0120plaque": 35127, "ocrat": 35128, "\u0120jackets": 35129, "\u0120HuffPost": 35130, "among": 35131, "\u0120confer": 35132, "487": 35133, "\u0120Lilly": 35134, "\u0120adapting": 35135, "\u0120Fay": 35136, "\u0120shoved": 35137, "vec": 35138, "\u0120refine": 35139, "\u0120gon": 35140, "\u0120gunmen": 35141, "zai": 35142, "\u0120Shuttle": 35143, "\u0120Izan": 35144, "\u01201913": 35145, "\u0120plethora": 35146, "\u00c2\u00b7\u00c2\u00b7": 35147, "\u0120510": 35148, "\u0120puberty": 35149, "\u0120241": 35150, "\u0120Wealth": 35151, "\u0120Alma": 35152, "\u0120MEM": 35153, "\u0120Adults": 35154, "Cas": 35155, "prison": 35156, "Race": 35157, "\u0120waterproof": 35158, "\u0120athleticism": 35159, "\u0120capitalize": 35160, "\u0120Juice": 35161, "\u0120illuminated": 35162, "\u0120Pascal": 35163, "\u0120irritation": 35164, "\u0120Witnesses": 35165, "adle": 35166, "\u0120Astro": 35167, "\u0120fax": 35168, "\u0120Elvis": 35169, "Primary": 35170, "\u0120Lich": 35171, "\u0120Elves": 35172, "\u0120residing": 35173, "\u0120stumble": 35174, "319": 35175, "\u0120PKK": 35176, "\u0120adversaries": 35177, "DOS": 35178, "\u0120Ritual": 35179, "\u0120smear": 35180, "\u0120arson": 35181, "idental": 35182, "\u0120scant": 35183, "\u0120monarchy": 35184, "\u0120halftime": 35185, "\u0120residue": 35186, "\u0120indign": 35187, "\u0120Shaun": 35188, "\u0120Elm": 35189, "auri": 35190, "Aff": 35191, "WATCH": 35192, "\u0120Lyon": 35193, "helps": 35194, "361": 35195, "\u0120lobbyist": 35196, "\u0120diminishing": 35197, "\u0120outbreaks": 35198, "\u0120goats": 35199, "favorite": 35200, "\u0120Nah": 35201, "sonian": 35202, "\u0120Booster": 35203, "\u0120sandbox": 35204, "\u0120Fare": 35205, "\u0120Malta": 35206, "\u0120attRot": 35207, "\u0120MOR": 35208, "lde": 35209, "\u0120navigating": 35210, "Touch": 35211, "\u0120untrue": 35212, "\u0120Disaster": 35213, "\u0120ludicrous": 35214, "Password": 35215, "\u0120JFK": 35216, "blogspot": 35217, "416": 35218, "\u0120UNDER": 35219, "ernal": 35220, "\u0120delaying": 35221, "TOP": 35222, "\u0120implants": 35223, "\u0120AVG": 35224, "\u0120Huge": 35225, "attr": 35226, "\u0120journalistic": 35227, "\u0120Peyton": 35228, "\u0120IA": 35229, "Rap": 35230, "goal": 35231, "\u0120Programme": 35232, "\u0120smashing": 35233, "wives": 35234, "println": 35235, "\u0120Plague": 35236, "inus": 35237, "EEP": 35238, "\u0120cruiser": 35239, "\u0120Parish": 35240, "uminium": 35241, "\u0120occupants": 35242, "\u0120Jihad": 35243, "mop": 35244, "\u0120pint": 35245, "\u0120hect": 35246, "\u0120Mecca": 35247, "director": 35248, "\u0120Funding": 35249, "\u0120Mixed": 35250, "\u0120stag": 35251, "Tier": 35252, "\u0120gust": 35253, "\u0120brightly": 35254, "orsi": 35255, "\u0120uphill": 35256, "RD": 35257, "\u0120lesions": 35258, "\u0120Bundy": 35259, "livious": 35260, "\u0120biologist": 35261, "\u0120Faculty": 35262, "\u0120Authorization": 35263, "\u0120244": 35264, "Allow": 35265, "\u00ef\u00b8": 35266, "\u0120Giul": 35267, "\u0120pertinent": 35268, "otaur": 35269, "esse": 35270, "\u0120Roof": 35271, "\u0120unmanned": 35272, "351": 35273, "\u0120Shak": 35274, "\u0120Orient": 35275, "\u0120endanger": 35276, "Dir": 35277, "\u0120replen": 35278, "edient": 35279, "\u0120tailor": 35280, "\u0120gadgets": 35281, "\u0120audible": 35282, "\u00e2\u013a\u0128": 35283, "Nice": 35284, "\u0120bombard": 35285, "\u0120Rape": 35286, "\u0120defiance": 35287, "\u0120TWO": 35288, "\u0120Filipino": 35289, "\u0120unaffected": 35290, "ervatives": 35291, "\u0120soared": 35292, "\u0120Bolton": 35293, "\u0120compromising": 35294, "\u0120Brewers": 35295, "RAL": 35296, "\u0120AHL": 35297, "icycle": 35298, "\u0120vampires": 35299, "\u0120dipped": 35300, "oyer": 35301, "\u0120XIII": 35302, "\u0120sideways": 35303, "\u0120Waste": 35304, "\u0120Diss": 35305, "\u0120\u00e2\u0136\u013e\u00e2\u0136\u0122\u00e2\u0136\u0122": 35306, "$.": 35307, "\u0120habitats": 35308, "\u0120Beef": 35309, "truth": 35310, "trained": 35311, "split": 35312, "Rus": 35313, "Andy": 35314, "\u0120Bram": 35315, "REP": 35316, "pid": 35317, "\u00e8\u00a3\u0127": 35318, "\u0120Mutant": 35319, "Anim": 35320, "\u0120Marina": 35321, "\u0120futile": 35322, "highest": 35323, "frequency": 35324, "\u0120epilepsy": 35325, "\u0120coping": 35326, "\u0120concise": 35327, "\u0120tracing": 35328, "\u0120SUN": 35329, "panel": 35330, "\u0120Sophie": 35331, "\u0120Crowley": 35332, "\u0120Adolf": 35333, "\u0120Shooter": 35334, "\u0120shaky": 35335, "\u0120IG": 35336, "\u0120Lies": 35337, "\u0120Barber": 35338, "pkg": 35339, "\u0120uptake": 35340, "\u0120predatory": 35341, "ULTS": 35342, "/**": 35343, "\u0120intoxicated": 35344, "\u0120Westbrook": 35345, "odder": 35346, "hement": 35347, "\u0120baseman": 35348, "APD": 35349, "storage": 35350, "\u0120Fifty": 35351, "editor": 35352, "GEN": 35353, "UTION": 35354, "irting": 35355, "\u0120sewing": 35356, "rift": 35357, "\u0120agony": 35358, "\u0120Sands": 35359, "\u0120254": 35360, "Cash": 35361, "\u0120lodge": 35362, "\u0120punt": 35363, "Natural": 35364, "\u0120Ideas": 35365, "\u0120erroneous": 35366, "\u0120Sensor": 35367, "\u0120Hannity": 35368, "\u01201921": 35369, "\u0120mould": 35370, "\u0120Gon": 35371, "kaya": 35372, "\u0120anonymously": 35373, "\u0120KEY": 35374, "\u0120simulator": 35375, "Winter": 35376, "\u0120streamed": 35377, "507": 35378, "?\",": 35379, "\u0120teased": 35380, "\u0120coefficient": 35381, "\u0120wartime": 35382, "\u0120THR": 35383, "''.": 35384, "\u0120Banking": 35385, "mpire": 35386, "\u0120fandom": 35387, "\u0120lia": 35388, "Ga": 35389, "\u0120downhill": 35390, "\u0120interpreting": 35391, "Individual": 35392, "Norm": 35393, "\u0120jealousy": 35394, "bitcoin": 35395, "\u0120pleasures": 35396, "\u0120Toys": 35397, "\u0120Chevrolet": 35398, "\u0120Advisor": 35399, "IZE": 35400, "\u0120receptions": 35401, "706": 35402, "Cro": 35403, "\u0120262": 35404, "\u0120citrus": 35405, "iru": 35406, "Reviewer": 35407, "jected": 35408, "UES": 35409, "anz": 35410, "1981": 35411, "\u0120Worker": 35412, "\u0120complied": 35413, "orescent": 35414, "continental": 35415, "Ton": 35416, "\u0120Prism": 35417, "\u0120Sheep": 35418, "\u0120288": 35419, "nox": 35420, "\u0120Vog": 35421, "Ord": 35422, "\u0120realms": 35423, "tek": 35424, "\u0120irrigation": 35425, "\u0120bicycles": 35426, "\u0120electronically": 35427, "poly": 35428, "tall": 35429, "());": 35430, "\u0120aesthetics": 35431, "\u0120Integrated": 35432, "Explore": 35433, "\u0120dunk": 35434, "476": 35435, "pain": 35436, "\u0120Jacques": 35437, "\u0120Dmit": 35438, "Frames": 35439, "\u0120reunited": 35440, "\u0120humid": 35441, "Dro": 35442, "Political": 35443, "\u0120youthful": 35444, "\u0120entails": 35445, "\u0120mosquito": 35446, "363": 35447, "species": 35448, "\u0120coordinating": 35449, "\u0120Mayhem": 35450, "\u0120Magnus": 35451, "Mount": 35452, "Improved": 35453, "\u0120STATE": 35454, "ATTLE": 35455, "\u0120flowed": 35456, "\u0120tackled": 35457, "\u0120fashioned": 35458, "\u0120reorgan": 35459, "ivari": 35460, "finger": 35461, "\u0120reluctantly": 35462, "etting": 35463, "\u0120Vand": 35464, "young": 35465, "\u0120Garland": 35466, "\u0120presumption": 35467, "\u0120amenities": 35468, "\u0120Pleasant": 35469, "onential": 35470, "\u0120Oxy": 35471, "\u0120morals": 35472, "\u0120Yah": 35473, "Ready": 35474, "Simon": 35475, "Enh": 35476, "Demon": 35477, "\u0120clich": 35478, "Monitor": 35479, "\u0120DU": 35480, "\u0120welcomes": 35481, "\u0120standout": 35482, "\u0120dreadful": 35483, "\u0120bananas": 35484, "\u0120balloons": 35485, "hooting": 35486, "basic": 35487, "\u0120suffix": 35488, "\u0120duly": 35489, "cano": 35490, "Chain": 35491, "atos": 35492, "\u0120geopolitical": 35493, "\u0120(&": 35494, "\u0120Gemini": 35495, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 35496, "\u0120acquitted": 35497, "Luck": 35498, "protect": 35499, "1024": 35500, "\u0120scarcity": 35501, "\u0120mindfulness": 35502, "ecided": 35503, "DN": 35504, "prime": 35505, "\u0120Presidents": 35506, "\u0120VIDEO": 35507, "\u0120(\u00e2\u012a\u0134": 35508, "addock": 35509, "NOR": 35510, "\u0120Pru": 35511, "pun": 35512, "\u0120LOL": 35513, "))))": 35514, "\u0120Liqu": 35515, "\u0120SAS": 35516, "\u0120styling": 35517, "\u0120punishments": 35518, "\u0120numb": 35519, "\u0120ascertain": 35520, "\u0120Rockies": 35521, "flu": 35522, "Thumbnail": 35523, "\u0120perpetrated": 35524, "\u0120Semi": 35525, "\u0120disarm": 35526, "\u0120Older": 35527, "\u0120Exception": 35528, "\u0120exponentially": 35529, "\u0120Communities": 35530, "\u0120abolish": 35531, "\u0120Partner": 35532, "ptoms": 35533, "\u0120777": 35534, "\u0120Foley": 35535, "\u0120Cases": 35536, "\u0120grease": 35537, "\u0120Rebirth": 35538, "Ground": 35539, "\u0120;)": 35540, "\u0120Doctrine": 35541, "ikini": 35542, "Ye": 35543, "\u0120Blossom": 35544, "\u0120persists": 35545, "bill": 35546, "\u0120infusion": 35547, "\u0120buddies": 35548, "911": 35549, "\u0120Patient": 35550, "\u0120demos": 35551, "\u0120acquaintance": 35552, "\u0120Paw": 35553, "atari": 35554, "\u0120xml": 35555, "\u0120fascination": 35556, "\u0120Serve": 35557, "\u00cf\u0124": 35558, "branded": 35559, "\u0120az": 35560, "Returns": 35561, "\u0120overshadow": 35562, "\u0120roam": 35563, "\u0120speedy": 35564, "numbered": 35565, "helial": 35566, "\u0120disciple": 35567, "\u0120assurances": 35568, "given": 35569, "pecting": 35570, "\u0120Natalie": 35571, "\u00e7\u0136\u00b0": 35572, "\u0120mosquitoes": 35573, "rotein": 35574, "\u0120numeric": 35575, "\u0120independents": 35576, "\u0120transitional": 35577, "\u0120reactionary": 35578, "\u0120Mechdragon": 35579, "doctor": 35580, "\u0120shortest": 35581, "\u0120sequential": 35582, "\u0120Bac": 35583, "\u0120Accounts": 35584, "\u00e3\u0123\u012e": 35585, "achy": 35586, "ractive": 35587, "\u0120Regiment": 35588, "\u0120breathtaking": 35589, "fficiency": 35590, "\u0120Bates": 35591, "\u0120311": 35592, "\u0120wardrobe": 35593, "fts": 35594, "\u0120Berk": 35595, "Simply": 35596, "\u0120Riverside": 35597, "ivering": 35598, "idential": 35599, "lucent": 35600, "\u0120enriched": 35601, "\u0120Conver": 35602, "\u0120Giving": 35603, "\u00e3\u0125\u013b": 35604, "\u0120legalize": 35605, "\u0120FTC": 35606, "\u0120freaking": 35607, "Mix": 35608, "\u0120terrestrial": 35609, "esian": 35610, "cients": 35611, "Wing": 35612, "LOAD": 35613, "\u0120ledge": 35614, "\u0120Violent": 35615, "\u0120Metall": 35616, "\u0120308": 35617, "\u0120southeastern": 35618, "hetto": 35619, "Meat": 35620, "\u0120slowdown": 35621, "\u0120retreated": 35622, "Jeremy": 35623, "endas": 35624, "*****": 35625, "eric": 35626, "\u0120reins": 35627, "oppable": 35628, "\u0120Humanity": 35629, "earances": 35630, "rigan": 35631, "Camera": 35632, "\u0120waivers": 35633, "soc": 35634, "\u0120alteration": 35635, "transform": 35636, "\u0120Cemetery": 35637, "506": 35638, "\u0120indefinite": 35639, "\u0120stimulating": 35640, "yg": 35641, "603": 35642, "\u0120Sop": 35643, "\u0120descriptive": 35644, "Phase": 35645, "\u0120Edmund": 35646, "\u0120pneumonia": 35647, "ventus": 35648, "Amb": 35649, "\u0120laboratories": 35650, "\u0120Exclusive": 35651, "ugar": 35652, "Were": 35653, "\u0120malfunction": 35654, "\u0120homosexuals": 35655, "\u0120-------": 35656, "uni": 35657, "\u0120turbines": 35658, "\u0120Equity": 35659, "Du": 35660, "\u0120minded": 35661, "\u0120RH": 35662, "\u0120Blackhawks": 35663, "\u0120feats": 35664, "\u01201700": 35665, "repl": 35666, "362": 35667, "laden": 35668, "\u0120indispensable": 35669, "lyss": 35670, "tti": 35671, "\u0120reel": 35672, "\u0120diverted": 35673, "\u0120likeness": 35674, "\u0120subscriptions": 35675, "\u0120fingert": 35676, "\u0120filthy": 35677, "destruct": 35678, "draft": 35679, "\u0120Bernardino": 35680, "launch": 35681, "\u0120perplex": 35682, "\u0120SUM": 35683, "carb": 35684, "\u0120sweater": 35685, "\u0120Venture": 35686, "\u0120Jag": 35687, "\u0120Celeb": 35688, "\u0120Voters": 35689, "\u0120steadfast": 35690, "\u0120athletics": 35691, "\u0120Hanson": 35692, "\u0120Drac": 35693, "Tracker": 35694, "\u0120commend": 35695, "\u0120Presidency": 35696, "\u0120DID": 35697, "informed": 35698, "\u0120webpage": 35699, "Pretty": 35700, "\u0120forcefully": 35701, "\u00e3\u0125\u0125\u00e3\u0124\u00af": 35702, "\u0120relocation": 35703, "\u0120satire": 35704, "\u00e2\u012b": 35705, "\u0120Sunderland": 35706, "\u00e6\u0126": 35707, "Voice": 35708, "????????": 35709, "\u0120informant": 35710, "\u0120bowel": 35711, "\u0120Uniform": 35712, "\u0120...\"": 35713, "\u0120purge": 35714, "\u0120picnic": 35715, "\u0120Umb": 35716, "\u0120UPDATE": 35717, "\u0120Sapphire": 35718, "\u0120Stall": 35719, "learn": 35720, "\u0120objectively": 35721, "\u0120obliter": 35722, "\u0120loophole": 35723, "\u0120journeys": 35724, "\u0120omission": 35725, "Pros": 35726, "\u0120Sidney": 35727, "ploma": 35728, "\u0120sprayed": 35729, "\u0120guru": 35730, "\u0120traitor": 35731, "\u0120timet": 35732, "\u0120snapping": 35733, "\u0120Sevent": 35734, "urnal": 35735, "\u0120Ukip": 35736, "\u0120bowed": 35737, "poral": 35738, "liberal": 35739, "Ros": 35740, "Questions": 35741, "iOS": 35742, "\u0120summarize": 35743, "STAT": 35744, "\u01201850": 35745, "apest": 35746, "\u0120lender": 35747, "\u0120Variable": 35748, "bringing": 35749, "\u0120LORD": 35750, ",)": 35751, "\u0120collapses": 35752, "xiety": 35753, "\u0120Ned": 35754, "YD": 35755, "\u0120Scha": 35756, "\u0120antibody": 35757, "\u0120disband": 35758, "yre": 35759, "illusion": 35760, "\u0120rover": 35761, "shed": 35762, "\u0120Hirosh": 35763, "cci": 35764, "\u0120calam": 35765, "\u0120Morton": 35766, "Pinterest": 35767, "\u01201928": 35768, "\u0120Euras": 35769, "ordes": 35770, "\u0120fences": 35771, "\u0120Inventory": 35772, "\u0120Valencia": 35773, "\u0120Ud": 35774, "\u0120Tiff": 35775, "\u0120sque": 35776, "\u0120quotation": 35777, "\u0120troublesome": 35778, "erker": 35779, "QUEST": 35780, "\u0120Kingdoms": 35781, "south": 35782, "\u0120levy": 35783, "Prince": 35784, "\u0120Sting": 35785, "\u0120nicknamed": 35786, "\u0120appe": 35787, "\u0120photographic": 35788, "\u0120corpus": 35789, "reference": 35790, "\u0120Trog": 35791, "Unt": 35792, ")=(": 35793, "\u0120Latvia": 35794, "\u0120activating": 35795, "\u0120licensee": 35796, "\u0120disparities": 35797, "\u0120Newsletter": 35798, "\u00e3\u0125\u0125\u00e3\u0125\u012a": 35799, "\u0120freeing": 35800, "\u0120Jeep": 35801, "\u0120Perception": 35802, "insk": 35803, "\u0120silicone": 35804, "\u0120Hayden": 35805, "Lean": 35806, "\u0120Suzuki": 35807, "ibrarian": 35808, "668": 35809, "\u0120spor": 35810, "\u0120correlations": 35811, "aghetti": 35812, "\u0120tuber": 35813, "\u0120IPCC": 35814, "ilus": 35815, "\u0120Vu": 35816, "\u0120wealthiest": 35817, "\u0120Carbuncle": 35818, "anza": 35819, "\u0120fooled": 35820, "\u0120Zur": 35821, "\u0120daddy": 35822, "rano": 35823, "ilian": 35824, "\u0120knockout": 35825, "fman": 35826, "required": 35827, "\u0120Wikileaks": 35828, "\u0120Duffy": 35829, "ONT": 35830, "\u0120insol": 35831, "\u0120Objects": 35832, "\u0120bou": 35833, "\u0120Nordic": 35834, "\u0120Insert": 35835, "scan": 35836, "\u0120dancers": 35837, "\u0120idiots": 35838, "majority": 35839, "\u0120Neville": 35840, "\u0120FreeBSD": 35841, "\u0120tart": 35842, "panic": 35843, "690": 35844, "\u0120cocoa": 35845, "\u0120sampled": 35846, "\u0120lookup": 35847, "Indust": 35848, "\u0120injections": 35849, "genre": 35850, "\u0120au": 35851, "\u0120roadway": 35852, "\u0120genitals": 35853, "Kind": 35854, "\u0120Examiner": 35855, "\u0120Yaz": 35856, "Fresh": 35857, "\u0120paralysis": 35858, "\u0120Aluminum": 35859, "\u0120reap": 35860, "ok\u00c3\u00a9": 35861, "\u0120sloppy": 35862, "\u0120Tunnel": 35863, "posium": 35864, "nery": 35865, "enic": 35866, "\u0120herbal": 35867, "\u0120Outer": 35868, "\u0120Builder": 35869, "\u0120incur": 35870, "\u0120ideologies": 35871, "\u0120backups": 35872, "consuming": 35873, "\u0120Detect": 35874, "deck": 35875, "\u0120KNOW": 35876, "\u0120Gret": 35877, "\u0120MIC": 35878, "\u0120toughness": 35879, "\u0120Exhibit": 35880, "\u0120hive": 35881, "Les": 35882, "\u0120SCHOOL": 35883, "\u0120Atari": 35884, "alde": 35885, "\u0120Null": 35886, "andestine": 35887, "mouse": 35888, "\u0120brigade": 35889, "489": 35890, "\u0120revol": 35891, "\u0120Lawson": 35892, "\u0120Wah": 35893, "opoly": 35894, "ebted": 35895, "\u0120Saunders": 35896, "\u0120313": 35897, "\u0120Winc": 35898, "\u0120taboo": 35899, "\u0120Helmet": 35900, "\u0120wedge": 35901, "chip": 35902, "\u0120Tina": 35903, "bg": 35904, "\u0120infuri": 35905, "rn": 35906, "\u0120anomalies": 35907, "\u0120Sync": 35908, "\u0120Exam": 35909, "\u0120Commit": 35910, "\u0120Diary": 35911, "\u0120ALSO": 35912, "\u0120Debor": 35913, "omedical": 35914, "\u0120comprehension": 35915, "655": 35916, "\u0120empowering": 35917, "\u0120ire": 35918, "\u0120juices": 35919, "\u0120ETH": 35920, "\u0120Boxing": 35921, "=\"/": 35922, "\u0120facilitated": 35923, "poke": 35924, "\u0120Parsons": 35925, "\u0120Moder": 35926, "travel": 35927, "\u0120civilizations": 35928, "\u0120libertarians": 35929, "\u0120rune": 35930, "\u0120Clarks": 35931, "athed": 35932, "\u0120campaigners": 35933, "\u0120Dispatch": 35934, "\u0120Fahrenheit": 35935, "\u0120Capcom": 35936, "----------": 35937, "\u0120lace": 35938, "\u0120draining": 35939, "\u0120liner": 35940, "\u0120Artificial": 35941, "\u00c3\u00a9n": 35942, "task": 35943, "]).": 35944, "\u0120GMO": 35945, "\u0120Operator": 35946, "ordinary": 35947, "\u0120Influence": 35948, "\u0120Ups": 35949, "\u0120potency": 35950, "ussen": 35951, "ospons": 35952, "\u0120Swim": 35953, "\u0120Deadline": 35954, "Unity": 35955, "\u0120culinary": 35956, "\u0120enlightenment": 35957, "\u0120wearer": 35958, "\u0120mined": 35959, "\u0120ply": 35960, "\u0120incest": 35961, "\u0120DVDs": 35962, "Walk": 35963, "BTC": 35964, "Trade": 35965, "\u0120deval": 35966, "iband": 35967, "\u0120Oversight": 35968, "Palestinian": 35969, "\u0120dart": 35970, "\u0120mul": 35971, "LR": 35972, "\u0120removable": 35973, "\u0120Realms": 35974, "\u00ec\u013f": 35975, "\u0120miscar": 35976, "\u0120Vulkan": 35977, "685": 35978, "\u00c3\u00a8re": 35979, "\u0120Sap": 35980, "\u0120merging": 35981, "\u0120Carly": 35982, "chester": 35983, "\u0120brisk": 35984, "\u0120luxurious": 35985, "\u0120Generator": 35986, "\u0120bitterness": 35987, "\u0120edible": 35988, "\u0120243": 35989, "TG": 35990, "\u0120rectangle": 35991, "WithNo": 35992, "below": 35993, "Jenn": 35994, "\u0120darkest": 35995, "\u0120hitch": 35996, "\u0120dosage": 35997, "\u0120scaven": 35998, "\u0120Keller": 35999, "\u0120Illustrated": 36000, "Certainly": 36001, "\u0120Mavericks": 36002, "Marginal": 36003, "\u0120diarrhea": 36004, "\u0120enormously": 36005, "\u0120999": 36006, "shr": 36007, "quart": 36008, "\u0120adamant": 36009, "\u0120Mew": 36010, "\u0120renovation": 36011, "\u0120cervical": 36012, "\u0120Percentage": 36013, "eners": 36014, "\u0120Kimber": 36015, "\u0120floats": 36016, "\u0120dex": 36017, "\u0120Witcher": 36018, "\u0120Swansea": 36019, "dm": 36020, "\u0120salty": 36021, "yellow": 36022, "\u0120cape": 36023, "\u0120Drain": 36024, "\u0120Paula": 36025, "\u0120Toledo": 36026, "lesi": 36027, "Magazine": 36028, "\u0120Wick": 36029, "\u0120Mn": 36030, "\u0120Ack": 36031, "\u0120Riding": 36032, "ASON": 36033, "\u0120homophobic": 36034, "ARP": 36035, "\u0120wandered": 36036, "CPU": 36037, "oodoo": 36038, "\u0120Pipe": 36039, "\u0120tightening": 36040, "\u0120Butt": 36041, "318": 36042, "\u0120deserted": 36043, "Session": 36044, "\u0120facilitating": 36045, "Jump": 36046, "\u0120emergencies": 36047, "OWER": 36048, "\u0120exhaustive": 36049, "\u0120AFTER": 36050, "\u0120heartbeat": 36051, "\u0120Label": 36052, "acky": 36053, "\u0120Certified": 36054, "iltration": 36055, "Ze": 36056, "\u0120Utt": 36057, "\u01201300": 36058, "\u0120presume": 36059, "\u0120Disp": 36060, "\u0120surged": 36061, "\u0120dolls": 36062, "Columb": 36063, "\u0120chimpan": 36064, "\u0120Razor": 36065, "\u0120ticks": 36066, "\u0120councillor": 36067, "\u0120pilgrimage": 36068, "\u0120Rebels": 36069, "\u0120QC": 36070, "\u0120Auction": 36071, "xia": 36072, "ikk": 36073, "bred": 36074, "\u0120insertion": 36075, "\u0120coarse": 36076, "dB": 36077, "SEE": 36078, "\u0120Zap": 36079, "\u0120Foo": 36080, "\u0120contempor": 36081, "\u0120Quarterly": 36082, "otions": 36083, "\u0120Alchemist": 36084, "\u0120Trey": 36085, "\u0120Duo": 36086, "Sweet": 36087, "804": 36088, "\u0120Giov": 36089, "\u0120funn": 36090, "Nin": 36091, "hoff": 36092, "\u0120ramifications": 36093, "\u01201922": 36094, "\u0120Experts": 36095, "azes": 36096, "\u0120garments": 36097, "arial": 36098, "\u0120Nab": 36099, "\u0120257": 36100, "\u0120Ved": 36101, "\u0120humorous": 36102, "\u0120Pompe": 36103, "\u0120nylon": 36104, "\u0120lurking": 36105, "\u0120Sergey": 36106, "\u0120Mattis": 36107, "\u0120misogyny": 36108, "\u0120Components": 36109, "\u0120Watching": 36110, "\u0120Folk": 36111, "ractical": 36112, "Bush": 36113, "\u0120taped": 36114, "\u0120grouping": 36115, "\u0120beads": 36116, "\u01202048": 36117, "\u0120condu": 36118, "querque": 36119, "Reading": 36120, "\u0120grievances": 36121, "Ultra": 36122, "\u0120endpoint": 36123, "Hig": 36124, "\u0120Static": 36125, "\u0120Scarborough": 36126, "Lua": 36127, "\u0120Messi": 36128, "aqu": 36129, "\u0120PsyNet": 36130, "\u0120Rudd": 36131, "\u0120avenue": 36132, "vp": 36133, "Jer": 36134, "\u0120shady": 36135, "\u0120Resist": 36136, "\u0120Artemis": 36137, "\u0120careless": 36138, "\u0120brokers": 36139, "\u0120temperament": 36140, "\u0120520": 36141, "Tags": 36142, "\u0120Turning": 36143, "\u0120uttered": 36144, "\u0120pedd": 36145, "\u0120improvised": 36146, "\u0120:(": 36147, "\u0120tabl": 36148, "\u0120plains": 36149, "1600": 36150, "pressure": 36151, "\u0120Essence": 36152, "margin": 36153, "friends": 36154, "\u0120Restoration": 36155, "\u0120pollut": 36156, "\u0120Poker": 36157, "\u0120Augustine": 36158, "\u0120CIS": 36159, "\u0120SEAL": 36160, "orama": 36161, "\u0120thwart": 36162, "seek": 36163, "\u0120pagan": 36164, "\u00c2\u00ba": 36165, "cpu": 36166, "\u0120garn": 36167, "\u0120assortment": 36168, "\u0120ILCS": 36169, "tower": 36170, "Recommended": 36171, "\u0120unborn": 36172, "\u0120RandomRedditor": 36173, "\u0120RandomRedditorWithNo": 36174, "\u0120paralyzed": 36175, "\u0120eruption": 36176, "\u0120intersect": 36177, "\u0120Stoke": 36178, "\u0120Sco": 36179, "Bind": 36180, "\u00e5\u00be": 36181, "\u0120PNG": 36182, "\u0120Negative": 36183, "\u0120NOAA": 36184, "Leon": 36185, "\u0120alloy": 36186, "\u0120Lama": 36187, "\u0120Diversity": 36188, "575": 36189, "\u0120underestimated": 36190, "\u0120Scor": 36191, "\u0120mural": 36192, "\u0120busted": 36193, "soon": 36194, "lif": 36195, "\u0120nonex": 36196, "\u0120allergy": 36197, "\u0120Underworld": 36198, "\u0120Rays": 36199, "\u0120Blasio": 36200, "\u0120hrs": 36201, "\u0120Dir": 36202, "\u0120327": 36203, "byter": 36204, "\u0120replacements": 36205, "\u0120activates": 36206, "rived": 36207, "MH": 36208, "\u0120pans": 36209, "\u0120HI": 36210, "\u0120longitudinal": 36211, "\u0120nuisance": 36212, "aler": 36213, "\u0120swell": 36214, "\u0120Signed": 36215, "sci": 36216, "\u0120Isles": 36217, "\u0120AGA": 36218, "\u0120defiant": 36219, "\u0120sonic": 36220, "ocon": 36221, "KC": 36222, "\u0120Aim": 36223, "tie": 36224, "ahah": 36225, "\u0120mL": 36226, "DX": 36227, "\u0120bisc": 36228, "\u0120Billboard": 36229, "\u0120SYSTEM": 36230, "NEY": 36231, "gaard": 36232, "\u0120distressed": 36233, "formerly": 36234, "Alan": 36235, "\u0120chefs": 36236, "\u0120optics": 36237, "\u0120Comet": 36238, "\u0120AMC": 36239, "\u0120redesigned": 36240, "irmation": 36241, "\u0120sightings": 36242, "382": 36243, "311": 36244, "\u0120WB": 36245, "\u0120contraction": 36246, "\u0120TOTAL": 36247, "Dual": 36248, "\u0120startled": 36249, "\u0120understandably": 36250, "\u0120sunglasses": 36251, "ETHOD": 36252, "\u0120docker": 36253, "\u0120surfing": 36254, "\u0120HEL": 36255, "\u0120Slack": 36256, "tones": 36257, "\u0120shalt": 36258, "Visual": 36259, "498": 36260, "Department": 36261, "cussion": 36262, "\u0120unrestricted": 36263, "\u0120tad": 36264, "\u0120rename": 36265, "employed": 36266, "\u0120educating": 36267, "\u0120grinned": 36268, "bedroom": 36269, "\u0120Activities": 36270, "\u0120Velvet": 36271, "\u0120SWAT": 36272, "\u0120shuffle": 36273, "igor": 36274, "\u0120saturation": 36275, "Finding": 36276, "cream": 36277, "icter": 36278, "\u0120vodka": 36279, "tracking": 36280, "tec": 36281, "\u0120foreground": 36282, "iesta": 36283, "\u0120vehement": 36284, "\u0120ECB": 36285, "\u0120Tie": 36286, "Ey": 36287, "\u0120turtles": 36288, "\u0120Railroad": 36289, "\u0120Katz": 36290, "\u0120Frames": 36291, "\u0120menace": 36292, "\u0120Fellowship": 36293, "\u0120Essential": 36294, "uggish": 36295, "\u0120drip": 36296, "chwitz": 36297, "\u0120Kyoto": 36298, "sb": 36299, "\u0120Nina": 36300, "Parameter": 36301, "\u0120alarms": 36302, "\u0120Claud": 36303, "\u0120pioneering": 36304, "\u0120chiefly": 36305, "\u0120Scream": 36306, "Collection": 36307, "\u0120thankfully": 36308, "\u0120Ronaldo": 36309, "\u00e5\u0143\u0132": 36310, "strip": 36311, "\u0120Disneyland": 36312, "commercial": 36313, "Seeing": 36314, "Soul": 36315, "\u0120evacuate": 36316, "\u0120civ": 36317, "\u0120Ashe": 36318, "\u0120divides": 36319, "\u0120Dagger": 36320, "rehensive": 36321, "\u0120berries": 36322, "\u0120DF": 36323, "\u0120sushi": 36324, "\u0120plurality": 36325, "WI": 36326, "\u0120disadvantaged": 36327, "\u0120battalion": 36328, "obiles": 36329, "451": 36330, "\u0120cling": 36331, "\u0120undeniable": 36332, "\u0120Lounge": 36333, "\u0120haunt": 36334, "phe": 36335, "\u0120quantify": 36336, "\u0120differed": 36337, "\u0120[*]": 36338, "\u0120Viz": 36339, "cum": 36340, "slave": 36341, "\u0120videog": 36342, "\u0120quar": 36343, "\u0120bundles": 36344, "\u0120Alonso": 36345, "tackle": 36346, "\u0120neuronal": 36347, "\u0120landslide": 36348, "confirmed": 36349, "\u0120Depth": 36350, "\u0120renewables": 36351, "Bear": 36352, "\u0120Macedonia": 36353, "\u0120jerseys": 36354, "\u0120bunk": 36355, "\u0120Spawn": 36356, "\u0120Controls": 36357, "\u0120Buchanan": 36358, "\u0120robotics": 36359, "\u0120emphasizing": 36360, "\u0120Tutorial": 36361, "hyp": 36362, "iston": 36363, "\u0120monumental": 36364, "\u00e6\u00b0": 36365, "\u0120Carry": 36366, "\u0120tbsp": 36367, "enance": 36368, "Hill": 36369, "arthed": 36370, "\u0120rotten": 36371, "Dean": 36372, "\u0120twisting": 36373, "\u0120goodwill": 36374, "\u0120immersion": 36375, "Living": 36376, "\u0120brushes": 36377, "\u0120CGI": 36378, "\u0120Atk": 36379, "traditional": 36380, "\u0120phantom": 36381, "\u0120Stamina": 36382, "\u0120expansions": 36383, "\u0120Marin": 36384, "\u0120embarked": 36385, "\u0120Eg": 36386, "intestinal": 36387, "\u0120PEOPLE": 36388, "\u0120Booth": 36389, "\u0120Appalach": 36390, "\u0120relegated": 36391, "VT": 36392, "MIT": 36393, "\u0120muster": 36394, "\u0120withdrawing": 36395, "\u0120microscope": 36396, "\u0120Gathering": 36397, "\u0120Crescent": 36398, "\u0120Argentine": 36399, "\u0120Decre": 36400, "\u0120Dominic": 36401, "\u0120buds": 36402, "antage": 36403, "\u0120Ion": 36404, "\u0120widened": 36405, "ONSORED": 36406, "\u0120Gloves": 36407, "iannopoulos": 36408, "razen": 36409, "feel": 36410, "\u0120repayment": 36411, "\u0120hindsight": 36412, "\u0120REALLY": 36413, "\u0120Pistol": 36414, "\u0120Brah": 36415, "\u0120watts": 36416, "\u0120survives": 36417, "\u0120flurry": 36418, "issy": 36419, "Alert": 36420, "\u0120Uruguay": 36421, "Phoenix": 36422, "Slow": 36423, "\u0120Grave": 36424, "\u0120Fir": 36425, "\u0120manageable": 36426, "\u0120tariff": 36427, "\u0120UDP": 36428, "\u0120Pistons": 36429, "\u0120Nigerian": 36430, "\u0120strikeouts": 36431, "\u0120cosmetics": 36432, "whelming": 36433, "fab": 36434, "cape": 36435, "proxy": 36436, "\u0120rethink": 36437, "\u0120overcoming": 36438, "simple": 36439, "\u0120woo": 36440, "\u0120distracting": 36441, "\u0120Stanton": 36442, "\u0120Tulsa": 36443, "\u0120Dock": 36444, "659": 36445, "\u0120discord": 36446, "\u0120Emacs": 36447, "\u0120Ves": 36448, "\u0120ROB": 36449, "\u0120reassuring": 36450, "\u0120consortium": 36451, "Muslims": 36452, "321": 36453, "\u0120prompts": 36454, "sei": 36455, "\u0120Hitch": 36456, "imposed": 36457, "\u0120Fool": 36458, "\u0120indiscrim": 36459, "wrong": 36460, "buquerque": 36461, "Davis": 36462, "!]": 36463, "\u0120timeless": 36464, "\u0120NEED": 36465, "\u0120pesticide": 36466, "\u0120rallying": 36467, "\u0120Calder": 36468, "\u0120\u00e5\u00a4": 36469, "\u0120xp": 36470, "\u0120Unle": 36471, "\u0120Export": 36472, "luaj": 36473, "Buff": 36474, ")[": 36937, "\u0120sqor": 36938, "Saudi": 36939, "\u0120istg": 36940, "\u0120indulge": 36941, "proc": 36942, "\u0120disgusted": 36943, "\u0120compounded": 36944, "\u0120nem": 36945, "\u0120schooling": 36946, "\u0120Cure": 36947, "processing": 36948, "Sol": 36949, "\u0120proverb": 36950, "itized": 36951, "\u0120Alvarez": 36952, "\u0120scarf": 36953, "\u0120rectangular": 36954, "reve": 36955, "\u0120hormonal": 36956, "\u0120Stress": 36957, "itizen": 36958, "\u0120425": 36959, "girls": 36960, "\u0120Noir": 36961, "\u0120Rapp": 36962, "\u0120marches": 36963, "church": 36964, "\u0120Uses": 36965, "\u0120405": 36966, "\u0120Berm": 36967, "\u0120ordinances": 36968, "\u0120Judgment": 36969, "Charges": 36970, "\u0120Zin": 36971, "\u0120dusty": 36972, "\u0120strawberries": 36973, "\u0120perce": 36974, "\u0120Thur": 36975, "\u0120Deborah": 36976, "netflix": 36977, "\u0120Lambert": 36978, "\u0120amused": 36979, "\u0120Guang": 36980, "YOU": 36981, "RGB": 36982, "\u0120CCTV": 36983, "\u0120fiat": 36984, "rang": 36985, "\u0120federation": 36986, "\u0120Mant": 36987, "\u0120Bust": 36988, "\u0120Mare": 36989, "respective": 36990, "\u0120Migration": 36991, "\u0120BIT": 36992, "590": 36993, "\u0120patriotism": 36994, "\u0120outlining": 36995, "region": 36996, "\u0120Jos\u00c3\u00a9": 36997, "\u0120blasting": 36998, "\u0120Ezra": 36999, "Bs": 37000, "\u0120undermines": 37001, "\u0120Smooth": 37002, "\u0120clashed": 37003, "radio": 37004, "\u0120transitioning": 37005, "\u0120Buccaneers": 37006, "\u0120Owl": 37007, "\u0120plugs": 37008, "\u0120hiatus": 37009, "\u0120Pinball": 37010, "\u0120mig": 37011, "\u0120Nutr": 37012, "\u0120Wolfe": 37013, "\u0120integers": 37014, "\u0120orbits": 37015, "\u0120Edwin": 37016, "\u0120DirectX": 37017, "bite": 37018, "\u0120blazing": 37019, "vr": 37020, "Edge": 37021, "\u0120PID": 37022, "exit": 37023, "\u0120Comed": 37024, "\u0120Pathfinder": 37025, "\u0120Guid": 37026, "\u0120Signs": 37027, "\u0120Zer": 37028, "\u0120Agenda": 37029, "\u0120reimbursement": 37030, "Mesh": 37031, "iPhone": 37032, "\u0120Marcos": 37033, "\u0120Sites": 37034, "hate": 37035, "enburg": 37036, "\u0120sockets": 37037, "pend": 37038, "Batman": 37039, "vir": 37040, "\u0120SHOW": 37041, "\u0120provisional": 37042, "conn": 37043, "\u0120Deaths": 37044, "ATIVE": 37045, "Profile": 37046, "sym": 37047, "JA": 37048, "\u0120ninja": 37049, "installed": 37050, "idates": 37051, "ebra": 37052, "\u0120Omaha": 37053, "\u0120seizing": 37054, "\u0120Beasts": 37055, "\u0120salts": 37056, "Mission": 37057, "Generally": 37058, "\u0120Trilogy": 37059, "heon": 37060, "legates": 37061, "\u0120dime": 37062, "\u0120faire": 37063, "parable": 37064, "Graph": 37065, "\u0120totaling": 37066, "\u0120diagrams": 37067, "\u0120Yanuk": 37068, "plet": 37069, "\u0120Meh": 37070, "\u0120mythical": 37071, "\u0120Stephens": 37072, "autical": 37073, "ochemistry": 37074, "\u0120kilograms": 37075, "\u0120elbows": 37076, "ancock": 37077, "\u0120BCE": 37078, "\u0120Prague": 37079, "\u0120improv": 37080, "\u0120Devin": 37081, "\u0120\"\\": 37082, "paralle": 37083, "\u0120supremacists": 37084, "\u0120Billion": 37085, "\u0120regimen": 37086, "innacle": 37087, "\u0120requisite": 37088, "angan": 37089, "\u0120Burlington": 37090, "ainment": 37091, "\u0120Objective": 37092, "omsky": 37093, "GV": 37094, "\u0120unilateral": 37095, "\u0120tc": 37096, "\u0120hires": 37097, "mental": 37098, "\u0120involuntary": 37099, "\u0120transpl": 37100, "\u0120ASCII": 37101, "\u00c2\u00a8": 37102, "Events": 37103, "\u0120doubted": 37104, "\u0120Kaplan": 37105, "\u0120Courage": 37106, "igon": 37107, "\u0120Managing": 37108, "\u0120Tart": 37109, "\u0120falsehood": 37110, "\u0120Violet": 37111, "\u0120airs": 37112, "\u0120fertilizer": 37113, "Britain": 37114, "\u0120aquatic": 37115, "ouf": 37116, "Words": 37117, "\u0120Hartford": 37118, "\u0120evenings": 37119, "\u0120Vengeance": 37120, "quite": 37121, "Gall": 37122, "\u0120Pret": 37123, "\u0120pdf": 37124, "\u0120LM": 37125, "\u0120Sochi": 37126, "\u0120Intercept": 37127, "920": 37128, "\u0120profitability": 37129, "\u0120Idle": 37130, "\u0120MacDonald": 37131, "\u0120Establishment": 37132, "umsy": 37133, "\u0120gatherings": 37134, "\u0120Naj": 37135, "Charlie": 37136, "\u0120ascent": 37137, "\u0120Protector": 37138, "\u0120algebra": 37139, "\u0120bios": 37140, "forums": 37141, "ELS": 37142, "Introduced": 37143, "\u0120335": 37144, "\u0120astronomy": 37145, "Contribut": 37146, "\u0120Polic": 37147, "Platform": 37148, "\u0120containment": 37149, "wrap": 37150, "\u0120coronary": 37151, "\u0120Jelly": 37152, "manager": 37153, "\u0120heartbreaking": 37154, "cair": 37155, "\u0120Chero": 37156, "cgi": 37157, "Medical": 37158, "\u0120Accountability": 37159, "!!\"": 37160, "ophile": 37161, "\u0120psychotic": 37162, "\u0120Restrict": 37163, "\u0120equitable": 37164, "issues": 37165, "\u01201905": 37166, "\u0120Nek": 37167, "cised": 37168, "\u0120Tracking": 37169, "\u0120ozone": 37170, "\u0120cooker": 37171, "rosis": 37172, "\u0120reopen": 37173, "\u0120infinity": 37174, "\u0120Pharmaceutical": 37175, "ensional": 37176, "Attempt": 37177, "\u0120Rory": 37178, "Marco": 37179, "\u0120awaits": 37180, "HOW": 37181, "treated": 37182, "\u0120bolst": 37183, "\u0120revered": 37184, "\u0120pods": 37185, "oppers": 37186, "0010": 37187, "\u0120amplitude": 37188, "rican": 37189, "SPONSORED": 37190, "\u0120trousers": 37191, "\u0120halves": 37192, "\u0120Kaine": 37193, "\u0120Cutler": 37194, "\u0120AUTH": 37195, "\u0120splendid": 37196, "\u0120preventive": 37197, "\u0120Dudley": 37198, "ifacts": 37199, "uminati": 37200, "\u0120Yin": 37201, "\u0120admon": 37202, "\u0120Vag": 37203, "\u0120inverted": 37204, "\u0120hastily": 37205, "\u0120Hague": 37206, "Lyn": 37207, "\u0120ledger": 37208, "\u0120astronomical": 37209, "getting": 37210, "\u0120circa": 37211, "\u0120Cic": 37212, "\u0120Tennis": 37213, "Limited": 37214, "\u0120dru": 37215, "\u0120BYU": 37216, "\u0120travellers": 37217, "\u0120pane": 37218, "\u0120Intro": 37219, "\u0120patiently": 37220, "\u0120aiding": 37221, "\u0120loos": 37222, "\u0120Tough": 37223, "\u0120293": 37224, "\u0120consumes": 37225, "SourceFile": 37226, "\u0120\"\"\"": 37227, "\u0120bonding": 37228, "\u0120tilted": 37229, "\u0120menstrual": 37230, "\u0120Celestial": 37231, "ULAR": 37232, "Plugin": 37233, "\u0120risking": 37234, "Naz": 37235, "\u0120Riyadh": 37236, "\u0120accredited": 37237, "\u0120skirm": 37238, "\u00e9\u013d": 37239, "\u0120examiner": 37240, "\u0120messing": 37241, "\u0120nearing": 37242, "\u0120Chern": 37243, "\u0120Beckham": 37244, "\u0120swapped": 37245, "\u0120goose": 37246, "Kay": 37247, "\u0120lofty": 37248, "\u0120Wallet": 37249, "\u0120['": 37250, "\u0120apocalypse": 37251, "\u0120bamboo": 37252, "\u0120SPACE": 37253, "\u0120Elena": 37254, "\u0120306": 37255, "acons": 37256, "\u0120tightened": 37257, "\u0120adolescence": 37258, "\u0120rainy": 37259, "\u0120vandalism": 37260, "\u0120Newtown": 37261, "\u0120conject": 37262, "cakes": 37263, "\u0120cheated": 37264, "\u0120moderators": 37265, "params": 37266, "EFF": 37267, "\u0120deceit": 37268, "\u0120STL": 37269, "\u0120Tanzania": 37270, "\u0120RI": 37271, "\u01201923": 37272, "\u0120Exile": 37273, "thel": 37274, "\u0120theolog": 37275, "\u0120quirky": 37276, "\u0120Irvine": 37277, "\u0120needy": 37278, "oris": 37279, "Um": 37280, "Ka": 37281, "\u0120mailbox": 37282, "322": 37283, "\u0120bos": 37284, "\u0120Petra": 37285, "KING": 37286, "\u0120enlarged": 37287, "Often": 37288, "\u0120badass": 37289, "\u0120343": 37290, "\u0120Places": 37291, "\u0120CAD": 37292, "\u0120pristine": 37293, "\u0120intervening": 37294, "direction": 37295, "\u0120laz": 37296, "\u0120DSM": 37297, "\u0120projecting": 37298, "\u0120Funk": 37299, "agog": 37300, "payment": 37301, "nov": 37302, "\u0120chatter": 37303, "ARB": 37304, "\u0120examinations": 37305, "\u0120Household": 37306, "\u0120Gus": 37307, "Ford": 37308, "414": 37309, "Boss": 37310, "\u0120mystic": 37311, "\u0120leaps": 37312, "\u0120Bav": 37313, "ulz": 37314, "budget": 37315, "Football": 37316, "\u0120subsidized": 37317, "\u0120firsthand": 37318, "\u0120coincide": 37319, "ocular": 37320, "Conn": 37321, "\u0120Collabor": 37322, "\u0120fools": 37323, "amura": 37324, "ahar": 37325, "rists": 37326, "\u0120swollen": 37327, "\u0120expended": 37328, "\u0120Pau": 37329, "sup": 37330, "\u0120spar": 37331, "\u0120keynote": 37332, "suff": 37333, "\u0120unequal": 37334, "\u0120progressing": 37335, "strings": 37336, "\u0120Gamergate": 37337, "Disney": 37338, "\u0120Eleven": 37339, "omnia": 37340, "\u0120scripted": 37341, "\u0120earners": 37342, "brother": 37343, "\u0120Enabled": 37344, "\u00e6\u00b3": 37345, "\u0120larvae": 37346, "\u0120LOC": 37347, "mess": 37348, "Wilson": 37349, "\u0120Template": 37350, "successfully": 37351, "\u0120paramount": 37352, "\u0120camouflage": 37353, "\u0120binds": 37354, "\u0120Quiet": 37355, "\u0120Shutterstock": 37356, "rush": 37357, "\u0120mascot": 37358, "fortune": 37359, "\u0120Colt": 37360, "\u0120Beyon": 37361, "habi": 37362, "\u0120hairc": 37363, "\u0120267": 37364, "\u0120Deus": 37365, "\u0120twitch": 37366, "\u0120concentrating": 37367, "\u0120nipples": 37368, "cible": 37369, "\u0120gir": 37370, "NZ": 37371, "Math": 37372, "nih": 37373, "Required": 37374, "\u0120ponder": 37375, "\u0120SAN": 37376, "\u0120weddings": 37377, "\u0120loneliness": 37378, "NES": 37379, "\u0120Mahjong": 37380, "695": 37381, "addle": 37382, "\u0120Garner": 37383, "\u0120COUR": 37384, "Bridge": 37385, "\u0120spree": 37386, "\u0120Caldwell": 37387, "\u0120bribery": 37388, "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 37389, "plugins": 37390, "\u0120racket": 37391, "\u0120champagne": 37392, "versible": 37393, "Vote": 37394, "\u0120modifiers": 37395, "Mayor": 37396, "680": 37397, "\u0120assemblies": 37398, "\u0120Sultan": 37399, "\u0120Ning": 37400, "\u0120Ladies": 37401, "\u0120sulfur": 37402, "\u0120orbs": 37403, "\u0120-----": 37404, "_______": 37405, "\u0120Journalism": 37406, "\u0120esports": 37407, "\u0120lush": 37408, "\u0120hue": 37409, "\u0120spectral": 37410, "Honest": 37411, "\u00e3\u0125\u0131": 37412, "\u0120bushes": 37413, "\u0120reinforcement": 37414, "\u0120reopened": 37415, "\u0120Wheels": 37416, "\u0120Morg": 37417, "rieving": 37418, "\u0120auxiliary": 37419, "\u0120jQuery": 37420, "\u0120BAT": 37421, "tesque": 37422, "\u0120vertex": 37423, "pure": 37424, "frey": 37425, "\u00e3\u0124\u00ba": 37426, "dos": 37427, "\u0120typh": 37428, "\u0120cull": 37429, "\u0120eq": 37430, "\u0120decon": 37431, "\u0120tossing": 37432, "\u0120disparate": 37433, "\u0120Brigham": 37434, "printf": 37435, "ledged": 37436, "\u0120sund": 37437, "\u0120cozy": 37438, "\u0120hepatitis": 37439, "performing": 37440, "\u0120aval": 37441, "\u0120GG": 37442, "future": 37443, "\u0120petertodd": 37444, "\u0120Kosovo": 37445, "\u0120magnets": 37446, "Already": 37447, "\u0120Edison": 37448, "\u0120Ceres": 37449, "\u0120RAID": 37450, "\u0120brilliance": 37451, "576": 37452, "\u0120derives": 37453, "\u0120hypertension": 37454, "\u0120\u00ce\u0136": 37455, "\u0120lambda": 37456, "\u0120flair": 37457, "\u0120missionaries": 37458, "\u0120rapes": 37459, "\u0120Starter": 37460, "\u0120Months": 37461, "\u0120defy": 37462, "\u0120seismic": 37463, "\u0120Raphael": 37464, "\u0120eurozone": 37465, "656": 37466, "zsche": 37467, "\u0120scratched": 37468, "\u0120bows": 37469, "\u0120Lennon": 37470, "\u0120Gaia": 37471, "\u0120dripping": 37472, "facts": 37473, "Ale": 37474, "\u0120frogs": 37475, "\u0120Breast": 37476, "ogeneity": 37477, "\u0120Prosecutor": 37478, "\u0120amplified": 37479, "\u0120Hodg": 37480, "\u0120Fn": 37481, "Thousands": 37482, "\u0120NIH": 37483, "\u0120Monitoring": 37484, "FTWARE": 37485, "\u0120Priebus": 37486, "\u0120Growing": 37487, "hunter": 37488, "\u0120diagnose": 37489, "\u0120Mald": 37490, "\u0120LR": 37491, "\u0120crowned": 37492, "\u0120bursting": 37493, "\u0120dissolution": 37494, "javascript": 37495, "\u0120usefulness": 37496, "\u0120Execution": 37497, ":(": 37498, "\u0120Ivory": 37499, "aah": 37500, "\u0120persecuted": 37501, "violence": 37502, "istas": 37503, "\u0120Crate": 37504, "\u0120impulses": 37505, "\u0120Spani": 37506, "edes": 37507, "Handle": 37508, "\u0120Zerg": 37509, "thinkable": 37510, "Lastly": 37511, "\u0120spontaneously": 37512, "\u0120inconvenient": 37513, "\u0120dismissing": 37514, "\u0120plotted": 37515, "\u0120eighty": 37516, "\u0120737": 37517, "rish": 37518, "\u0120Thornton": 37519, "atham": 37520, "\u0120sitcom": 37521, "Ven": 37522, "Recipe": 37523, "tel": 37524, "lund": 37525, "\u0120clears": 37526, "\u0120Sasuke": 37527, "\u0120258": 37528, "\u0120opting": 37529, "\u0120enraged": 37530, "esthetic": 37531, "\u0120Ae": 37532, "uchs": 37533, "Prep": 37534, "Flow": 37535, "\u0120runoff": 37536, "\u0120Eating": 37537, "\u0120Giles": 37538, "\u0120Acting": 37539, "resources": 37540, "ibaba": 37541, "\u0120rpm": 37542, "\u0120skewed": 37543, "\u0120Blanc": 37544, "\u0120Sakuya": 37545, "\u0120hotter": 37546, "\u01201924": 37547, "opian": 37548, "cko": 37549, "\u0120crumbling": 37550, "\u0120captains": 37551, "\u0120Appropriations": 37552, "leaders": 37553, "dropping": 37554, "anuts": 37555, "\u0120reversing": 37556, "\u0120Pose": 37557, "\u0120Sek": 37558, "Scot": 37559, "\u0120Idea": 37560, "cise": 37561, "\u0120Slovenia": 37562, "\u0120317": 37563, "Doctor": 37564, "\u0120crocod": 37565, "aldi": 37566, "Sea": 37567, "\u0120Farrell": 37568, "\u0120mercenaries": 37569, "\u0120RNC": 37570, "\u0120Guess": 37571, "\u0120pacing": 37572, "Machine": 37573, "StreamerBot": 37574, "\u0120Charity": 37575, "\u0120298": 37576, "\u0120cannons": 37577, "\u0120Toby": 37578, "TPPStreamerBot": 37579, "\u0120Passion": 37580, "cfg": 37581, "Thom": 37582, "\u0120badges": 37583, "\u0120Bernstein": 37584, ".\u00e2\u0122\u0135": 37585, "\u0120POP": 37586, "\u0120Conj": 37587, "\u0120initialization": 37588, "\u0120biodiversity": 37589, "Dub": 37590, "\u0120feudal": 37591, "\u0120disclaimer": 37592, "\u0120crow": 37593, "\u0120ignition": 37594, "arf": 37595, "SHA": 37596, "\u0120kHz": 37597, "hazard": 37598, "\u0120Artists": 37599, "oeuv": 37600, "679": 37601, "\u0120Rudy": 37602, "Nine": 37603, "\u0120Ramadan": 37604, "\u00e5\u00bd": 37605, "itto": 37606, "\u0120adrenaline": 37607, "Cert": 37608, "\u0120smelled": 37609, "\u0120impunity": 37610, "\u0120agendas": 37611, "\u0120Reborn": 37612, "\u0120Concent": 37613, "\u0120Seems": 37614, "\u0120omega": 37615, "\u0120Dustin": 37616, "\u0120backer": 37617, "\u0120Sauce": 37618, "\u0120Boyle": 37619, "WIN": 37620, "\u0120spins": 37621, "\u0120pauses": 37622, "upt": 37623, "\u0120shredded": 37624, "\u0120strapped": 37625, "\u0120Corruption": 37626, "\u0120scratches": 37627, "\u0120ni": 37628, "\u0120attire": 37629, "\u0120SAF": 37630, "FactoryReloaded": 37631, "\u0120IPS": 37632, "\u0120(%": 37633, "\u0120seminar": 37634, "focus": 37635, "civil": 37636, "\u01201860": 37637, "intosh": 37638, "\u0120continual": 37639, "\u0120abbrevi": 37640, "\u0120Sok": 37641, "ocobo": 37642, "XM": 37643, "\u0120frantic": 37644, "\u0120unavoidable": 37645, "\u0120artery": 37646, "\u0120annotations": 37647, "bath": 37648, "Climate": 37649, "\u0120dors": 37650, "\u0120Slide": 37651, "coord": 37652, "\u0120Reload": 37653, "\u0120LDL": 37654, "\u0120Lovecraft": 37655, "\u0120unimagin": 37656, "\u0120resembled": 37657, "\u0120barracks": 37658, "np": 37659, "\u0120surrogate": 37660, "\u0120categorized": 37661, "\u00e3\u0124\u00a9": 37662, "\u0120vaccinated": 37663, "\u0120drainage": 37664, "\u0120indist": 37665, "\u0120WhatsApp": 37666, "\u01201870": 37667, "olerance": 37668, "invoke": 37669, "amorph": 37670, "\u0120reconnect": 37671, "\u0120emanc": 37672, "\u0120blindness": 37673, "\u01201280": 37674, "internet": 37675, "collar": 37676, "\u0120altru": 37677, "\u0120abyss": 37678, "\u0120TRI": 37679, "657": 37680, "\u0120infused": 37681, "HEAD": 37682, "\u0120forestry": 37683, "\u0120Woody": 37684, "\u0120Ci": 37685, "wi": 37686, "sam": 37687, "784": 37688, "holiday": 37689, "\u0120mogul": 37690, "\u0120Fees": 37691, "\u0120DEN": 37692, "Internal": 37693, "urbed": 37694, "fusc": 37695, "atom": 37696, "\u0120Illusion": 37697, "\u0120polled": 37698, "\u0120flap": 37699, "\u0120coax": 37700, "LGBT": 37701, "Analy": 37702, "\u0120Sections": 37703, "\u0120Californ": 37704, "emn": 37705, "\u0120hither": 37706, "\u0120NIGHT": 37707, "\u0120nailed": 37708, "\u0120Pipeline": 37709, "391": 37710, "oof": 37711, "\u0120Primal": 37712, "verend": 37713, "\u0120slashing": 37714, "\u0120retri": 37715, "aviour": 37716, "\u0120departing": 37717, "gil": 37718, "ISC": 37719, "\u0120midway": 37720, "\u0120ultrasound": 37721, "\u0120behaving": 37722, "\u0120Tara": 37723, "classes": 37724, "Virtual": 37725, "\u0120Colonial": 37726, "\u0120stripping": 37727, "\u0120orchestrated": 37728, "\u0120Graves": 37729, "452": 37730, "\u0120Ironically": 37731, "\u0120Writers": 37732, "\u0120lends": 37733, "\u0120Manz": 37734, "\u0120raven": 37735, "\u0120oxidative": 37736, "\u0120266": 37737, "ELF": 37738, "actually": 37739, "ascar": 37740, "Draft": 37741, "\u0120favourable": 37742, "\u0120humiliating": 37743, "\u0120fidelity": 37744, "\u0120Hof": 37745, "\u0120Xuan": 37746, "496": 37747, "\u0120layered": 37748, "atis": 37749, "790": 37750, "\u0120paycheck": 37751, "iton": 37752, "Kar": 37753, "\u0120VMware": 37754, "\u0120Farmer": 37755, "\u0120servic": 37756, "glomer": 37757, "\u0120slump": 37758, "\u0120Fabric": 37759, "\u0120DOC": 37760, "esting": 37761, "\u0120reassure": 37762, "\u0120phyl": 37763, "volt": 37764, "itory": 37765, "Rules": 37766, "\u0120oxidation": 37767, "\u0120prized": 37768, "\u0120mistress": 37769, "\u0120Django": 37770, "WARN": 37771, "\u00e5\u0133": 37772, "\u0120encode": 37773, "\u0120Feedback": 37774, "\u0120stupidity": 37775, "Ian": 37776, "\u0120Yugoslavia": 37777, "\u00d7\u00a8": 37778, "acl": 37779, "UTE": 37780, "1977": 37781, "\u0120qualifies": 37782, "\u0120pulses": 37783, "pretty": 37784, "\u0120froze": 37785, "\u0120ss": 37786, "Iterator": 37787, "\u0120urgently": 37788, "\u0120mailed": 37789, "\u0120Cham": 37790, "\u0120sustaining": 37791, "\u0120basil": 37792, "\u0120puppies": 37793, "ilant": 37794, "\u0120PLEASE": 37795, "lap": 37796, "aceous": 37797, "Fear": 37798, "\u0120Mastery": 37799, "automatic": 37800, "\u0120TAG": 37801, "\u0120antim": 37802, "agles": 37803, "473": 37804, "frames": 37805, "\u0120whispers": 37806, "\u0120Whoever": 37807, "\u0120bravery": 37808, "\u0120UKIP": 37809, "ractions": 37810, "\"\"\"": 37811, "\u0120tame": 37812, "\u0120parted": 37813, "everything": 37814, "CONT": 37815, "\u0120indebted": 37816, "\u0120addr": 37817, "rek": 37818, "IRED": 37819, "\u0120eminent": 37820, "clinton": 37821, "\u0120ousted": 37822, "\u0120reviewer": 37823, "\u0120meltdown": 37824, "\u0120rearr": 37825, "\u0120Yao": 37826, "thereal": 37827, "abyte": 37828, "\u0120stumbling": 37829, "\u0120batches": 37830, "\u0120259": 37831, "\u0120contraceptive": 37832, "\u0120prostitute": 37833, "ensis": 37834, "Decl": 37835, "\u0120Strikes": 37836, "Military": 37837, "\u0120Oath": 37838, "vacc": 37839, "ppings": 37840, "052": 37841, "\u0120partName": 37842, "amping": 37843, "Reports": 37844, "KI": 37845, "CHR": 37846, "\u0120subtly": 37847, "swers": 37848, "Blake": 37849, "usual": 37850, "\u0120contestants": 37851, "\u0120cartridges": 37852, "\u0120GREAT": 37853, "\u0120blush": 37854, "\u0120\u00e2\u0122\u00ba": 37855, "472": 37856, "\u0120reasoned": 37857, "\u00e3\u0125\u00a4": 37858, "paralleled": 37859, "\u0120dyn": 37860, "agate": 37861, "\u0120nightly": 37862, "\u00e5\u0128": 37863, "556": 37864, "\u0120semantic": 37865, "\u0120Advoc": 37866, "\u0120!!": 37867, "\u0120disagrees": 37868, "\u0120BW": 37869, "Veh": 37870, "\u0120harming": 37871, "\u0120embraces": 37872, "\u0120strives": 37873, "\u0120inland": 37874, "\u0120Kard": 37875, "\u0120heats": 37876, "\u0120Ginny": 37877, "utan": 37878, "ernaut": 37879, "ylene": 37880, "\u0120Elev": 37881, "JD": 37882, "\u0120hars": 37883, "\u0120Starr": 37884, "\u0120skysc": 37885, "\u0120collaborators": 37886, "Usually": 37887, "\u0120revolutions": 37888, "\u0120STATS": 37889, "\u0120dismantle": 37890, "\u0120confidently": 37891, "\u0120kinetic": 37892, "Ali": 37893, "\u0120percentile": 37894, "\u0120extracting": 37895, "illian": 37896, "estead": 37897, "\u0120physicists": 37898, "\u0120Marshal": 37899, "\u0120fellowship": 37900, "\u0120dashed": 37901, "\u0120UR": 37902, "\u0120Sioux": 37903, "\u0120Compact": 37904, "amide": 37905, "Python": 37906, "\u0120Leigh": 37907, "\u0120Pharmac": 37908, "istrates": 37909, "herical": 37910, "\u0120fue": 37911, "\u0120Emin": 37912, "\u0120({": 37913, "\u0120Neighborhood": 37914, "\u0120disrupting": 37915, "\u0120Dup": 37916, "\u0120gland": 37917, "\u0120Sev": 37918, "\u0120Marian": 37919, "argon": 37920, "\u0120Dund": 37921, "\u0120": 46904, "\u0120Philips": 46905, "\u0120Kafka": 46906, "\u0120upheaval": 46907, "\u0120sentimental": 46908, "\u0120sax": 46909, "\u0120Akira": 46910, "serial": 46911, "Matrix": 46912, "\u0120electing": 46913, "\u0120commenter": 46914, "\u0120Nebula": 46915, "plets": 46916, "\u0120Nadu": 46917, "\u0120Adren": 46918, "\u0120enshr": 46919, "\u0120RAND": 46920, "financial": 46921, "\u0120Clyde": 46922, "utherford": 46923, "\u0120signage": 46924, "\u0120deline": 46925, "\u0120phosphate": 46926, "roversial": 46927, "fascist": 46928, "\u0120Vall": 46929, "\u0120Bethlehem": 46930, "\u0120fors": 46931, "\u0120english": 46932, "Solid": 46933, "Nature": 46934, "\u0120va": 46935, "\u0120Guests": 46936, "\u0120tantal": 46937, "\u0120autoimmune": 46938, ";;;;;;;;;;;;": 46939, "\u0120Totally": 46940, "\u0120Ov": 46941, "\u0120defences": 46942, "\u0120Coconut": 46943, "\u0120tranquil": 46944, "\u0120ploy": 46945, "\u0120flavours": 46946, "\u0120Flask": 46947, "\u00e3\u0124\u00a8\u00e3\u0125\u00ab": 46948, "\u0120Weston": 46949, "\u0120Volvo": 46950, "870": 46951, "\u0120microphones": 46952, "verbal": 46953, "RPG": 46954, "\u0120iii": 46955, ";}": 46956, "028": 46957, "\u0120headlined": 46958, "\u0120primed": 46959, "\u0120hoard": 46960, "\u0120Shad": 46961, "\u0120ENTER": 46962, "\u0120triangular": 46963, "\u0120capit": 46964, "lik": 46965, "\u0120Ancients": 46966, "\u0120lash": 46967, "\u0120convol": 46968, "\u0120colonel": 46969, "enemy": 46970, "Gra": 46971, "\u0120pubs": 46972, "utters": 46973, "\u0120assigns": 46974, "\u0120Penet": 46975, "\u0120Monstrous": 46976, "\u0120Bowen": 46977, "ilver": 46978, "Haunted": 46979, "\u0120Ding": 46980, "started": 46981, "plin": 46982, "\u0120contaminants": 46983, "\u0120DOE": 46984, "ffen": 46985, "\u0120Technician": 46986, "Ry": 46987, "\u0120robbers": 46988, "\u0120hotline": 46989, "\u0120Guardiola": 46990, "\u0120Kaufman": 46991, "rower": 46992, "\u0120Dresden": 46993, "\u0120Alpine": 46994, "Elf": 46995, "\u0120fmt": 46996, "\u0120Sard": 46997, "urses": 46998, "gpu": 46999, "Unix": 47000, "\u0120unequivocally": 47001, "\u0120Citizenship": 47002, "quad": 47003, "mire": 47004, "\u0120Sweeney": 47005, "Battery": 47006, "615": 47007, "\u0120pancakes": 47008, "\u0120oats": 47009, "Maps": 47010, "\u0120Contrast": 47011, "mbudsman": 47012, "\u0120EPS": 47013, "\u0120subcommittee": 47014, "\u0120sourcing": 47015, "\u0120sizing": 47016, "\u0120Buffer": 47017, "\u0120Mandatory": 47018, "\u0120moderates": 47019, "\u0120Patterns": 47020, "\u0120Chocobo": 47021, "\u0120Zan": 47022, "\u0120STATES": 47023, "\u0120Judging": 47024, "\u0120Inher": 47025, "*:": 47026, "\u0120bil": 47027, "\u0120Yen": 47028, "\u0120exhilar": 47029, "ollower": 47030, "zers": 47031, "\u0120snug": 47032, "maximum": 47033, "\u0120despicable": 47034, "\u0120PACK": 47035, "\u0120Annex": 47036, "\u0120sarcastic": 47037, "\u0120latex": 47038, "\u0120tamp": 47039, "\u0120Sao": 47040, "bah": 47041, "\u0120Reverend": 47042, "\u0120Chinatown": 47043, "\u0120AUT": 47044, "documented": 47045, "\u0120GABA": 47046, "\u0120Canaan": 47047, "\u0120\u00d9\u0127": 47048, "\u0120governs": 47049, "prev": 47050, "Esc": 47051, "\u0120Estimates": 47052, "OSP": 47053, "\u0120endeavour": 47054, "\u0120Closing": 47055, "ometime": 47056, "everyone": 47057, "\u0120worsen": 47058, "\u0120scanners": 47059, "\u0120deviations": 47060, "\u0120Robotics": 47061, "\u0120Compton": 47062, "\u0120sorcerer": 47063, "\u0120endogenous": 47064, "\u0120emulation": 47065, "\u0120Piercing": 47066, "\u0120Aph": 47067, "\u0120Socket": 47068, "\u0120bould": 47069, "\u0120OU": 47070, "\u0120Borderlands": 47071, "\u01201863": 47072, "Gordon": 47073, "\u0120WTO": 47074, "\u0120restricts": 47075, "\u0120mosaic": 47076, "\u0120melodies": 47077, "\u00e7\u0126": 47078, "Tar": 47079, "\u0120disson": 47080, "\u0120Provides": 47081, "\u0120......": 47082, "bek": 47083, "FIX": 47084, "\u0120broom": 47085, "anship": 47086, "Doctors": 47087, "\u0120nerds": 47088, "\u0120Regions": 47089, "naissance": 47090, "\u0120mete": 47091, "\u0120crept": 47092, "plings": 47093, "\u0120girlfriends": 47094, "knit": 47095, "igent": 47096, "owe": 47097, "\u0120ushered": 47098, "\u0120Baz": 47099, "Mobil": 47100, "434": 47101, "\u0120Presents": 47102, "origin": 47103, "\u0120insomnia": 47104, "\u0120Aux": 47105, "439": 47106, "\u0120Chili": 47107, "irsch": 47108, "GAME": 47109, "\u0120gestation": 47110, "algia": 47111, "romising": 47112, "$,": 47113, "crow": 47114, "\u0120Inspection": 47115, "atomic": 47116, "Relations": 47117, "JOHN": 47118, "roman": 47119, "\u0120Clockwork": 47120, "\u0120Bakr": 47121, "mone": 47122, "MET": 47123, "\u0120thirsty": 47124, "\u0120bc": 47125, "\u0120faculties": 47126, "Rum": 47127, "\u0120nuance": 47128, "\u0120Darius": 47129, "pleting": 47130, "fters": 47131, "etchup": 47132, "Registration": 47133, "\u0120KE": 47134, "Rah": 47135, "\u0120preferential": 47136, "\u0120Lash": 47137, "\u0120HH": 47138, "Valid": 47139, "\u0120NAV": 47140, "\u0120starve": 47141, "\u0120Gong": 47142, "zynski": 47143, "\u0120Actress": 47144, "\u0120wik": 47145, "\u0120unaccompanied": 47146, "lvl": 47147, "Bride": 47148, "ADS": 47149, "\u0120Commando": 47150, "\u0120Vaughn": 47151, "Wallet": 47152, "\u0120hopping": 47153, "\u0120Vie": 47154, "\u0120caveats": 47155, "\u0120alas": 47156, "ifled": 47157, "abuse": 47158, "661": 47159, "\u0120ibn": 47160, "\u0120gul": 47161, "\u0120robbing": 47162, "til": 47163, "ILA": 47164, "\u0120mitigating": 47165, "\u0120aptly": 47166, "\u0120tyrant": 47167, "\u0120midday": 47168, "\u0120Gilmore": 47169, "\u0120Decker": 47170, "\u0120\u00c2\u00a7\u00c2\u00a7": 47171, "partial": 47172, "Exactly": 47173, "\u0120phenotype": 47174, "\u0120[+]": 47175, "\u0120Plex": 47176, "\u0120Ips": 47177, "versions": 47178, "\u0120ebook": 47179, "\u0120chic": 47180, "gross": 47181, "\":\"\"},{\"": 47182, "\u0120Surprisingly": 47183, "Morgan": 47184, "\u0120residues": 47185, "\u0120Confederation": 47186, "infeld": 47187, "\u0120lyr": 47188, "moderate": 47189, "\u0120perpendicular": 47190, "VK": 47191, "\u0120synchronized": 47192, "\u0120refreshed": 47193, "\u0120adore": 47194, "\u0120Torment": 47195, "olina": 47196, "\u01202600": 47197, "ItemTracker": 47198, "\u0120pies": 47199, "\u0120FAT": 47200, "\u0120RHP": 47201, "048": 47202, "\u0120RESP": 47203, "\u0120BJ": 47204, "allows": 47205, "Pand": 47206, "\u0120unwelcome": 47207, "\u0120Voc": 47208, "\u0120Bastard": 47209, "\u0120OW": 47210, "\u0120LAR": 47211, "\u0120Healer": 47212, "Environmental": 47213, "\u0120Kenyan": 47214, "\u0120Trance": 47215, "\u0120Pats": 47216, "\u0120aliases": 47217, "\u0120Garfield": 47218, "\u0120campaigner": 47219, "\u0120advancements": 47220, "\u0120Okinawa": 47221, "\u0120Coh": 47222, "owsky": 47223, "\u0120starved": 47224, "\u0120sizeable": 47225, "\u0120:-)": 47226, "\u0120mRNA": 47227, "\u0120suspensions": 47228, "istar": 47229, "Scotland": 47230, "Prin": 47231, "------------------------------------------------": 47232, "\u0120502": 47233, "\u0120teaspoons": 47234, "\u01201050": 47235, "\u0120coercive": 47236, "\u0120Masonic": 47237, "edded": 47238, "\u0120Passenger": 47239, "\u0120latt": 47240, "\u0120braces": 47241, "\u0120Steal": 47242, "\u0120NYT": 47243, "\u0120Kats": 47244, "\u0120Celest": 47245, "aez": 47246, "Tu": 47247, "\u0120Coulter": 47248, "\u00f0\u0141\u013a": 47249, "Flickr": 47250, "\u0120Wilmington": 47251, "iths": 47252, "++;": 47253, "\u0120vending": 47254, "\u0120negro": 47255, "\u0120Phi": 47256, "\u0120Yellowstone": 47257, "Callback": 47258, "\u0120shampoo": 47259, "\u0120Shades": 47260, "wat": 47261, "\u0120superhuman": 47262, "\u0120ridiculed": 47263, "\u0120holiest": 47264, "ombo": 47265, "\u0120interns": 47266, "\u0120hone": 47267, "\u0120Paragu": 47268, "URI": 47269, "\u0120dangling": 47270, "\u00e3\u0124\u00bb": 47271, "sov": 47272, "ictional": 47273, "availability": 47274, "\u0120revocation": 47275, "\u0120dow": 47276, "inic": 47277, "\u0120THEIR": 47278, "\u0120iso": 47279, "\u0120outings": 47280, "\u0120Lethal": 47281, "\u0120)))": 47282, "\u0120inaccur": 47283, "\u0120outlandish": 47284, "\u0120anus": 47285, "letico": 47286, "idon": 47287, "lol": 47288, "\u0120unregulated": 47289, "\u0120succumbed": 47290, "\u0120cuff": 47291, "\u0120Wasteland": 47292, "letal": 47293, "\u0120substr": 47294, "\u0120coffers": 47295, "\u0120automakers": 47296, "ovi": 47297, "\u0120Xue": 47298, "\u0120Daytona": 47299, "\u0120jarring": 47300, "\u0120fumes": 47301, "\u0120disbanded": 47302, "zik": 47303, "itton": 47304, "\u0120strikingly": 47305, "\u0120spores": 47306, "Adapter": 47307, ".):": 47308, "\u0120Lyndon": 47309, "ivalry": 47310, "\u0120orally": 47311, "\u0120tumultuous": 47312, "\u0120displeasure": 47313, "\u0120cones": 47314, "orrect": 47315, "\u0120appease": 47316, "\u0120derby": 47317, "\u0120Tripoli": 47318, "\u0120Aless": 47319, "\u0120poked": 47320, "\u0120Guilty": 47321, "vP": 47322, "Enough": 47323, "\u0120originals": 47324, "699": 47325, "\u0120rabbi": 47326, "\u0120proverbial": 47327, "\u0120postpone": 47328, "elope": 47329, "\u0120Misty": 47330, "\u0120staffed": 47331, "\u0120Unemployment": 47332, "reditary": 47333, "\u0120diligent": 47334, "recomm": 47335, "measures": 47336, "asin": 47337, "825": 47338, "\u0120ponds": 47339, "\u0120mmol": 47340, "\u0120SAR": 47341, "\u0120CARE": 47342, "\u0120371": 47343, "\u0120clenched": 47344, "\u0120Corsair": 47345, "\u0120caricature": 47346, "zn": 47347, "attach": 47348, "\u0120Schro": 47349, "speak": 47350, "painted": 47351, "\u0120Suc": 47352, "\u0120ENT": 47353, "\u0120cellul": 47354, "\u0120Paid": 47355, "diagn": 47356, "WHERE": 47357, "\u0120texted": 47358, "Barn": 47359, "\u0120retracted": 47360, "\u0120Referred": 47361, "Sav": 47362, "\u0120upkeep": 47363, "\u0120workplaces": 47364, "\u0120Tokens": 47365, "\u0120amplify": 47366, "clinical": 47367, "\u0120multic": 47368, "mberg": 47369, "\u0120convoluted": 47370, "Region": 47371, "565": 47372, "\u0120Topic": 47373, "\u0120snail": 47374, "\u0120saline": 47375, "\u0120insurrection": 47376, "\u0120Petr": 47377, "forts": 47378, "BAT": 47379, "\u0120Navajo": 47380, "\u0120rudimentary": 47381, "\u0120Laksh": 47382, "ONDON": 47383, "Measure": 47384, "\u0120transformer": 47385, "\u0120Goddard": 47386, "\u0120coincides": 47387, "irin": 47388, "Rex": 47389, "\u0120Bok": 47390, "quit": 47391, "\u0120shotguns": 47392, "\u0120proletarian": 47393, "\u0120scorp": 47394, "\u0120Ada": 47395, "514": 47396, "\u0120slander": 47397, "recorded": 47398, "\u0120embell": 47399, "risome": 47400, "\u0120apologizing": 47401, "\u0120Mulcair": 47402, "\u0120Gibraltar": 47403, "Cla": 47404, "\u0120allot": 47405, "\u0120Attention": 47406, "\u0120433": 47407, "leave": 47408, "\u0120whine": 47409, "\u0120Issa": 47410, "\u0120Faust": 47411, "\u0120Barron": 47412, "heny": 47413, "\u0120victimized": 47414, "Jews": 47415, "\u0120nurturing": 47416, "ettel": 47417, "Winged": 47418, "\u0120Subtle": 47419, "\u0120flavorful": 47420, "\u0120Reps": 47421, "enged": 47422, "callback": 47423, "\u0120directional": 47424, "\u0120clasp": 47425, "\u0120Directions": 47426, "planet": 47427, "iculture": 47428, "Helper": 47429, "icion": 47430, "acia": 47431, "\u0120\u00e7\u00a5\u0140": 47432, "\u0120surges": 47433, "\u0120canoe": 47434, "\u0120Premiership": 47435, "been": 47436, "\u0120defied": 47437, "\u0120Trooper": 47438, "\u0120tripod": 47439, "\u0120gasp": 47440, "\u0120Euph": 47441, "\u0120Ads": 47442, "vernight": 47443, "highly": 47444, "Role": 47445, "\u0120entangled": 47446, "\u0120Zeit": 47447, "618": 47448, "\u0120Rusty": 47449, "\u0120havens": 47450, "\u0120Vaughan": 47451, "HAEL": 47452, "\u0120SERVICE": 47453, "/,": 47454, "\u0120stricken": 47455, "\u0120delusions": 47456, "\u0120bis": 47457, "\u0120Haf": 47458, "\u0120gratification": 47459, "\u0120enticing": 47460, "UNCH": 47461, "Adams": 47462, "\u0120OLED": 47463, "\u0120Beetle": 47464, "\u01201899": 47465, "\u0120SOFTWARE": 47466, "ategor": 47467, "VL": 47468, "\u0120Totem": 47469, "\u0120Gators": 47470, "ATURES": 47471, "\u0120impedance": 47472, "Registered": 47473, "\u0120Cary": 47474, "\u0120Aerial": 47475, "onne": 47476, "enium": 47477, "\u0120dred": 47478, "\u0120Beg": 47479, "\u0120concurrently": 47480, "\u0120superpower": 47481, "\u0120Xan": 47482, "jew": 47483, "imester": 47484, "\u0120Dickinson": 47485, "\u00e2\u0136\u0123": 47486, "Fla": 47487, "\u0120pree": 47488, "\u0120Rollins": 47489, "\u00a9\u00b6\u00e6": 47490, "\u0120denomination": 47491, "\u0120Lana": 47492, "516": 47493, "\u0120inciting": 47494, "scribed": 47495, "juries": 47496, "\u0120Wonders": 47497, "approximately": 47498, "\u0120suspending": 47499, "\u0120mountainous": 47500, "\u0120Laugh": 47501, "oidal": 47502, "Ns": 47503, "Detect": 47504, ")=": 47505, "\u0120Luthor": 47506, "\u0120Schwarzenegger": 47507, "\u0120Muller": 47508, "\u0120Devi": 47509, "ecycle": 47510, "Jar": 47511, "613": 47512, "\u0120Longh": 47513, "Bah": 47514, "\u0120SPORTS": 47515, "nw": 47516, "\u0120refinement": 47517, "\u0120waterways": 47518, "\u0120diner": 47519, "Blade": 47520, "683": 47521, "Fac": 47522, "\u0120initials": 47523, "\u0120rog": 47524, "\u0120paranormal": 47525, "BUT": 47526, "\u0120[(": 47527, "\u0120Swanson": 47528, "\u0120Mesh": 47529, "\u00e2\u0138\u00ac": 47530, "Improve": 47531, "\u0120Radiation": 47532, "\u0120Esther": 47533, "\u0120Esk": 47534, "\u0120Aly": 47535, "iky": 47536, "\u0120irrad": 47537, "\u0120Buckingham": 47538, "\u0120refill": 47539, "\u0120._": 47540, "Repe": 47541, "CONCLUS": 47542, "\u0120differentiated": 47543, "\u0120chirop": 47544, "\u0120Atkins": 47545, "Pattern": 47546, "\u0120excise": 47547, "\u0120cabal": 47548, "NSA": 47549, "\u0120STA": 47550, "\u0120SIL": 47551, "\u0120Paraly": 47552, "\u0120rye": 47553, "\u0120Howell": 47554, "\u0120Countdown": 47555, "nesses": 47556, "alysed": 47557, "\u0120resize": 47558, "\u00e3\u0124\u00bd": 47559, "\u0120budgetary": 47560, "\u0120Stras": 47561, "wang": 47562, "\u0120apiece": 47563, "\u0120precincts": 47564, "\u0120peach": 47565, "\u0120skyline": 47566, "\u0120353": 47567, "popular": 47568, "Appearances": 47569, "\u0120Mechanics": 47570, "\u0120DevOnline": 47571, "Sullivan": 47572, "Zen": 47573, "\u0120pu": 47574, "opolis": 47575, "544": 47576, "\u0120deform": 47577, "\u0120counteract": 47578, "\u0120Lange": 47579, "\u0120417": 47580, "Console": 47581, "774": 47582, "\u0120nodding": 47583, "\u0120populism": 47584, "\u0120hep": 47585, "\u0120counselling": 47586, "compliance": 47587, "UFF": 47588, "\u0120undeniably": 47589, "\u0120railing": 47590, "\u0120Horowitz": 47591, "\u0120Simone": 47592, "\u0120Bungie": 47593, "\u0120ak": 47594, "\u0120Talks": 47595, "xff": 47596, "flake": 47597, "Crash": 47598, "\u0120sweaty": 47599, "\u0120banquet": 47600, "\u0120OFFIC": 47601, "\u0120inventive": 47602, "\u0120astronomer": 47603, "\u0120Stamford": 47604, "\u0120Scare": 47605, "\u0120GREEN": 47606, "olicited": 47607, "\u0120rusher": 47608, "\u0120centrist": 47609, "ighting": 47610, "\u0120subclass": 47611, "\u0120disav": 47612, "\u0120defund": 47613, "\u0120Nanto": 47614, "ociate": 47615, "mast": 47616, "\u0120pacif": 47617, "\u0120mend": 47618, "eers": 47619, "immigration": 47620, "ESSION": 47621, "\u0120numbering": 47622, "\u0120laughable": 47623, "\u0120Ended": 47624, "viation": 47625, "emark": 47626, "Pitt": 47627, "\u0120meticulous": 47628, "\u0120LF": 47629, "\u0120congratulated": 47630, "\u0120Birch": 47631, "\u0120swayed": 47632, "\u0120semifinals": 47633, "\u0120humankind": 47634, "matter": 47635, "\u0120Equip": 47636, "opausal": 47637, "Said": 47638, "\u0120Layout": 47639, "\u0120voicing": 47640, "\u0120thug": 47641, "\u0120pornographic": 47642, "IPS": 47643, "\u0120moaning": 47644, "\u0120grievance": 47645, "\u0120confessions": 47646, "escal": 47647, "TEXTURE": 47648, "Authent": 47649, "osaurus": 47650, "Purchase": 47651, "\u0120relegation": 47652, "alter": 47653, "\u0120\u00c2\u0142\u00c2\u0142": 47654, "\u0120riddled": 47655, "\u0120ogre": 47656, "\u0120Lowell": 47657, "Occup": 47658, "Eat": 47659, "\u0120Hyder": 47660, "\u0120Adviser": 47661, "Commerce": 47662, "Hunt": 47663, "\u0120Orth": 47664, "\u0120Competitive": 47665, "\u0120CLA": 47666, "CDC": 47667, "\u0120salads": 47668, "Fle": 47669, "\u0120industrialized": 47670, "`,": 47671, "\u0120OWN": 47672, "\u0120beck": 47673, "\u0120Particularly": 47674, "oubt": 47675, "\u0120mM": 47676, "\u0120Hussain": 47677, "\u0120Chennai": 47678, "\u0120920": 47679, "\u0120appointing": 47680, "\u0120Cullen": 47681, ",,,,,,,,": 47682, "\u0120pores": 47683, "verified": 47684, "\u0120biochemical": 47685, "emate": 47686, "\u0120cowardly": 47687, "\u0120Helsinki": 47688, "\u0120Ethiopian": 47689, "SOURCE": 47690, "ERC": 47691, "estro": 47692, "\u0120biotech": 47693, "\u0120Sour": 47694, "\u0120brewer": 47695, "Bloomberg": 47696, "\u0120intensify": 47697, "Glass": 47698, "anco": 47699, "\u0120FDR": 47700, "greSQL": 47701, "\u0120Fires": 47702, "\u00a9\u00b6\u00e6\u00a5\u00b5": 47703, "eco": 47704, "1001": 47705, "\u0120Homeless": 47706, "\u0120instantaneous": 47707, "\u0120Haste": 47708, "igel": 47709, "Diamond": 47710, "\u0120paving": 47711, "\u0120landfill": 47712, "\u0120dads": 47713, "houn": 47714, ":]": 47715, "\u0120incendiary": 47716, "\u0120Livingston": 47717, "\u0120Hilbert": 47718, "\u0120Checks": 47719, "styles": 47720, "inators": 47721, "\u0120Clive": 47722, "phrine": 47723, "\u0120chimpanzees": 47724, "\u0120pall": 47725, "\u0120JM": 47726, "\u0120Aadhaar": 47727, "\u00f0\u013f": 47728, "\u0120achievable": 47729, "disabled": 47730, "PET": 47731, "OOOOOOOO": 47732, "Mot": 47733, "\u0120intangible": 47734, "\u0120ballet": 47735, "\u0120Webs": 47736, "\u0120Estimated": 47737, "Effects": 47738, "\u0120bailed": 47739, "Joshua": 47740, "\u0120turbulence": 47741, "\u0120occupant": 47742, "\u0120Daylight": 47743, "\u0120361": 47744, "meet": 47745, "\u0120statically": 47746, "\u0120onlook": 47747, "\u0120ki": 47748, "illegal": 47749, "\u0120velvet": 47750, "\u0120dehydration": 47751, "\u0120acquies": 47752, "\u0120Rez": 47753, "akura": 47754, "\u0120Upton": 47755, "atro": 47756, "\u0120incomprehensible": 47757, "\u0120backdoor": 47758, "\u0120Rhino": 47759, "727": 47760, "\u0120maths": 47761, ")+": 47762, "\u0120heresy": 47763, "\u0120df": 47764, "\u0120Roche": 47765, "\u0120Lydia": 47766, "\u0120pancreat": 47767, "reply": 47768, "arrell": 47769, "\u0120solicitation": 47770, "\u0120circadian": 47771, "BIP": 47772, "\u0120foray": 47773, "\u0120cryptic": 47774, "izu": 47775, "imeo": 47776, "\u0120Tomato": 47777, "\u0120Homs": 47778, "examination": 47779, "\u0120quarry": 47780, "\u0120Valiant": 47781, "\u0120Jericho": 47782, "\u0120INCLUD": 47783, "\u01201840": 47784, "519": 47785, "\u0120resists": 47786, "\u0120snapshots": 47787, "\u0120Spur": 47788, "\u0120Antiqu": 47789, "Login": 47790, "\u0120bestselling": 47791, "\u0120antic": 47792, "\u0120Sutherland": 47793, "\u00e3\u0124\u00a2\u00e3\u0125\u00ab": 47794, "\u0120~/": 47795, "\u0120Parm": 47796, "\u00e8\u0125": 47797, "Pages": 47798, "intensity": 47799, "\u0120immobil": 47800, "\u01201865": 47801, "zzo": 47802, "\u0120nifty": 47803, "\u0120fentanyl": 47804, "\u0120Preservation": 47805, "ophen": 47806, "\u0120darts": 47807, "\u0120Dinosaur": 47808, "pointers": 47809, "\u0120Rite": 47810, "suggest": 47811, "awareness": 47812, "\u0120Sheridan": 47813, "\u0120stances": 47814, "\u0120sorcery": 47815, "\u0120perjury": 47816, "\u0120Nikola": 47817, "iever": 47818, "\u0120fiance": 47819, "\u0120Jordanian": 47820, "\u0120Balloon": 47821, "\u0120nab": 47822, "\u0120kb": 47823, "\u0120humanities": 47824, "\u0120Tanaka": 47825, "hillary": 47826, "\u0120consultancy": 47827, "\u0120Zub": 47828, "\u0120remission": 47829, "\u0120confid": 47830, "CHQ": 47831, "\u0120Fug": 47832, "\u0120improvis": 47833, "Yep": 47834, "/_": 47835, "\u0120unwillingness": 47836, "\u0120portfolios": 47837, "055": 47838, "\u0120Instructor": 47839, "aiman": 47840, "\u0120claimants": 47841, "Mbps": 47842, "\u0120Bye": 47843, "received": 47844, "Tweet": 47845, "\u0120indemn": 47846, "riz": 47847, "amara": 47848, "Nat": 47849, "\u0120evaluates": 47850, "\u0120Lur": 47851, "epad": 47852, "FOX": 47853, "\u0120Thro": 47854, "\u0120rusty": 47855, "\u0120bedrock": 47856, "\u0120Oprah": 47857, "JB": 47858, "\u0120manipulative": 47859, "\u0120willful": 47860, "\u0120relapse": 47861, "\u0120extant": 47862, "Theme": 47863, "Sensor": 47864, "\u0120Stability": 47865, "govern": 47866, "\u0120poppy": 47867, "\u0120knack": 47868, "\u0120insulated": 47869, "\u0120Tile": 47870, "\u0120Extrem": 47871, "\u0120untold": 47872, "\u0120converge": 47873, "\u0120refuel": 47874, "igroup": 47875, "\u0120distortions": 47876, "\u0120ravaged": 47877, "\u0120mechanically": 47878, "\u0120Reilly": 47879, "\u0120Nose": 47880, "\u0120Incarnation": 47881, "\u0120Becky": 47882, "abbling": 47883, "\u0120taco": 47884, "\u0120rake": 47885, "\u0120melancholy": 47886, "\u0120illustrious": 47887, "\u0120Dartmouth": 47888, "Guide": 47889, "\u0120Razer": 47890, "\u0120Benz": 47891, "Ultimate": 47892, "\u0120Surprise": 47893, "\u0120pageant": 47894, "offer": 47895, "Whoever": 47896, "\u0120wiser": 47897, "\u0120chemist": 47898, "\u0120HELL": 47899, "\u0120Bulk": 47900, "\u0120plutonium": 47901, "\u0120COVER": 47902, "\u00d6\u00bc": 47903, "failed": 47904, "\u0120tirelessly": 47905, "\u0120infertility": 47906, "\u0120Trident": 47907, "\u0120Showtime": 47908, "\u0120Civ": 47909, "Vice": 47910, "requires": 47911, "ittance": 47912, "\u0120uncontrolled": 47913, "interesting": 47914, "561": 47915, "\u0120innovate": 47916, "ategic": 47917, "Lie": 47918, "\u0120Selling": 47919, "Ul": 47920, "\u0120savior": 47921, "\u0120Tosh": 47922, "\u0120swast": 47923, "PASS": 47924, "\u0120rink": 47925, "\u0120cardio": 47926, "\u0120Iro": 47927, "udi": 47928, "\u0120vantage": 47929, "\u0120vans": 47930, "\u0120Ni\u00c3\u00b1o": 47931, "+=": 47932, "\u0120propagate": 47933, "": 49029, "\u0120leukemia": 49030, "\u0120eluc": 49031, "\u0120announcer": 49032, "\u0120Lithuan": 49033, "\u0120Armageddon": 49034, "\u00e5\u0129": 49035, "Lenin": 49036, "\u0120Ruk": 49037, "\u0120pepp": 49038, "\u0120Romantic": 49039, "\u0120PIT": 49040, "\u0120Interstellar": 49041, "\u0120Atkinson": 49042, "Raid": 49043, "Js": 49044, "Goal": 49045, "Course": 49046, "\u0120vanishing": 49047, "esley": 49048, "\u0120Rounds": 49049, "Elsa": 49050, "593": 49051, "\u0120redundancy": 49052, "\u0120STAND": 49053, "\u0120prophetic": 49054, "\u0120habitable": 49055, "ryu": 49056, "\u0120faintly": 49057, "MODE": 49058, "\u0120flanked": 49059, "IRC": 49060, "Awesome": 49061, "\u0120spurious": 49062, "\u0120Zah": 49063, "\u0120MSG": 49064, "\u0120shading": 49065, "\u0120motivational": 49066, "\u0120Santana": 49067, "\u0120SPR": 49068, "\u0120excruciating": 49069, "omial": 49070, "\u0120Miko": 49071, "\u0120Leopard": 49072, "Abyss": 49073, "\u0120[|": 49074, "dirty": 49075, "\u0120baths": 49076, "\u0120demoral": 49077, "andre": 49078, "PB": 49079, "\u0120unification": 49080, "\u0120sacrament": 49081, "\u0120[&": 49082, "\u0120priceless": 49083, "\u0120gelatin": 49084, "\u0120emanating": 49085, "\u0120Allaah": 49086, "986": 49087, "\u0120outburst": 49088, "\u0120eras": 49089, "\u0120XVI": 49090, "\u0120SPI": 49091, "Ott": 49092, "\u0120Lazarus": 49093, "PLIED": 49094, "Flying": 49095, "blogs": 49096, "Wisconsin": 49097, "Raven": 49098, "\u0120rebate": 49099, "\u0120creeps": 49100, "\u0120Span": 49101, "\u0120Painter": 49102, "\u0120Kira": 49103, "\u0120Amos": 49104, "\u0120Corvette": 49105, "Consumer": 49106, "\u0120Recover": 49107, "cki": 49108, "\u0120pesky": 49109, "\u0120Invention": 49110, "Companies": 49111, "\u0120challengers": 49112, "ademic": 49113, "\u0120Ukrainians": 49114, "\u0120Neurolog": 49115, "\u0120Forsaken": 49116, "\u0120entrants": 49117, "\u0120embattled": 49118, "\u0120defunct": 49119, "\u0120Glacier": 49120, "\u0120poisons": 49121, "\u0120Horses": 49122, "makes": 49123, "\u0120Dirt": 49124, "\u0120423": 49125, "hhh": 49126, "\u0120Transformation": 49127, "QUIRE": 49128, "..................": 49129, "\u0120traveller": 49130, "\u0120Sexy": 49131, "\u0120Kern": 49132, "ipolar": 49133, "\u0120ransomware": 49134, "oooooooooooooooo": 49135, "Ec": 49136, "ruby": 49137, "Professional": 49138, "\u0120Outbreak": 49139, "argument": 49140, "Grey": 49141, "\u0120Fifa": 49142, "\u0120CHO": 49143, "\u0120FORM": 49144, "\u0120Amtrak": 49145, "-[": 49146, "\u0120cradle": 49147, "\u0120antioxidants": 49148, "\u00e3\u0123\u00ae\u00e5\u00ae": 49149, "736": 49150, "\u0120NASL": 49151, "\u0120Contributions": 49152, "Indiana": 49153, "\u0120STEP": 49154, "CSS": 49155, "\u0120salient": 49156, "\u0120allocations": 49157, "yrights": 49158, "\u0120mashed": 49159, "\u0120Cutter": 49160, "Sexual": 49161, "\u0120pounded": 49162, "\u0120fanbase": 49163, "\u0120casc": 49164, "\u0120Transparency": 49165, "\u0120analytic": 49166, "\u0120Summoner": 49167, "\u00d7\u0140": 49168, "\u0120ADC": 49169, "detail": 49170, "\u0120vanquished": 49171, "\u0120crabs": 49172, "arie": 49173, "Destroy": 49174, "\u0120Sack": 49175, "\u0120transistor": 49176, "Alabama": 49177, "\u0120Koen": 49178, "\u0120Fisheries": 49179, "cone": 49180, "\u0120annexed": 49181, "\u0120MGM": 49182, "esa": 49183, "\u0120faked": 49184, "\u0120Congratulations": 49185, "\u0120hindered": 49186, "\u0120correctional": 49187, "\u0120ITV": 49188, "leeve": 49189, "\u0120inappropriately": 49190, "licks": 49191, "\u0120trespass": 49192, "\u0120paws": 49193, "\u0120negotiator": 49194, "\u0120Christensen": 49195, "limits": 49196, "\u0120Dianne": 49197, "\u0120elegance": 49198, "\u0120Contracts": 49199, "anke": 49200, "Obj": 49201, "\u0120vigilance": 49202, "\u0120castles": 49203, "\u0120NAD": 49204, "\u0120Holo": 49205, "\u0120emphatically": 49206, "\u0120Titus": 49207, "\u0120Serving": 49208, "\u0120Richie": 49209, "\u0120Pigs": 49210, "568": 49211, "\u0120animosity": 49212, "\u0120Attributes": 49213, "\u0120Uriel": 49214, "MQ": 49215, "myra": 49216, "\u0120Applicant": 49217, "\u0120psychiatrists": 49218, "\u0120Vij": 49219, "\u0120Abby": 49220, "agree": 49221, "Push": 49222, "\u0120kWh": 49223, "hiba": 49224, "\u0120incite": 49225, "\u0120Weasley": 49226, "\u0120Taxi": 49227, "ministic": 49228, "hyper": 49229, "\u0120Farn": 49230, "\u0120601": 49231, "\u0120Nationwide": 49232, "Fake": 49233, "952": 49234, "\u0120maize": 49235, "\u0120interacted": 49236, "\u0120transitioned": 49237, "\u0120parasitic": 49238, "\u0120harmonic": 49239, "\u0120decaying": 49240, "\u0120baseless": 49241, "nsics": 49242, "\u0120transpired": 49243, "\u0120abundantly": 49244, "\u0120Forensic": 49245, "\u0120treadmill": 49246, "\u0120Jav": 49247, "aband": 49248, "\u0120sshd": 49249, "\u0120frontman": 49250, "\u0120Jakarta": 49251, "oller": 49252, "drops": 49253, "\u0120SERVICES": 49254, "romptu": 49255, "ophical": 49256, "hospital": 49257, "bledon": 49258, "645": 49259, "\u0120midrange": 49260, "\u0120EVENT": 49261, "culated": 49262, "rawled": 49263, "\u0120perched": 49264, "\u0120overboard": 49265, "\u0120Peel": 49266, "\u0120Pwr": 49267, "\u0120Carth": 49268, "\u0120COMPLE": 49269, "coe": 49270, "shall": 49271, "\u0120deterrence": 49272, "METHOD": 49273, "\u0120Absent": 49274, "MEN": 49275, "\u0120sill": 49276, "\u0120LEVEL": 49277, "York": 49278, "\u0120sinners": 49279, "\u0120OPEC": 49280, "\u0120Nur": 49281, "\u0120Designs": 49282, "selection": 49283, "\u0120unworthy": 49284, "CHA": 49285, "\u0120strengthens": 49286, "883": 49287, "edly": 49288, "\u0120slicing": 49289, "\u0120malnutrition": 49290, "\u0120filmmaking": 49291, "\u0120Polk": 49292, "urated": 49293, "\u0120421": 49294, "breakers": 49295, "!'\"": 49296, "\u0120wetlands": 49297, "\u0120Discrimination": 49298, "\u0120allowable": 49299, "\u0120steered": 49300, "\u0120Sicily": 49301, "SAM": 49302, "\u0120mustache": 49303, "\u0120mids": 49304, "\u0120clipped": 49305, "\u0120circulate": 49306, "\u0120brittle": 49307, "\u0120Buildings": 49308, "raised": 49309, "\u0120Roundup": 49310, "\u0120wealthier": 49311, "\u0120overwrite": 49312, "\u0120overpowered": 49313, "\u0120Gerrard": 49314, "sites": 49315, "PDATED": 49316, "\u0120acutely": 49317, "\u0120Gamble": 49318, "\u0120pim": 49319, "\u0120Kus": 49320, "Typically": 49321, "Deploy": 49322, "\u0120Moroccan": 49323, "potion": 49324, "combe": 49325, "\u0120vigilante": 49326, "\u0120363": 49327, "Stew": 49328, "\u0120Bagg": 49329, "\u0120resided": 49330, "\u0120Spo": 49331, "\u0120remnant": 49332, "\u0120emptiness": 49333, "brainer": 49334, "\u0120outpatient": 49335, "priority": 49336, "\u0120leptin": 49337, "\u0120Payton": 49338, "\u0120Gleaming": 49339, "\u0120Shed": 49340, "\u0120Polo": 49341, "\u0120Mormonism": 49342, "restricted": 49343, "arlane": 49344, "wx": 49345, "\u0120creatine": 49346, "\u0120Anon": 49347, "\u0120STUD": 49348, "\u0120JUL": 49349, "\u0120Tee": 49350, "528": 49351, "089": 49352, "\u0120hatched": 49353, "Dispatch": 49354, "\u0120Composite": 49355, "\u0120451": 49356, "puff": 49357, "\u0120XCOM": 49358, "\u0120Orn": 49359, "\u0120THANK": 49360, "ENDED": 49361, "\u0120Asheville": 49362, "\u0120\u00c3\u013e": 49363, "\u0120mango": 49364, "\u0120Slightly": 49365, "worldly": 49366, "\u0120Wander": 49367, "\u0120Expand": 49368, "\u0120Chr": 49369, "Mist": 49370, "\u0120orthodoxy": 49371, "\u0120UNESCO": 49372, "regate": 49373, "Elsewhere": 49374, "kie": 49375, "irled": 49376, "\u0120topple": 49377, "\u0120adoptive": 49378, "\u0120Legs": 49379, "dress": 49380, "\u0120Sagan": 49381, "bare": 49382, "\u0120Glou": 49383, "Crunch": 49384, "\u0120helpers": 49385, "\u0120chronically": 49386, "\u0120Huma": 49387, "10000": 49388, "\u0120accommodating": 49389, "\u00e4\u00ba\u0136": 49390, "\u0120wrinkles": 49391, "\u0120dodged": 49392, "fourth": 49393, "\u0120precon": 49394, "\u0120compressor": 49395, "\u0120Kare": 49396, "\u0120evict": 49397, "\u0120Warwick": 49398, "imar": 49399, "\u0120modernization": 49400, "\u0120bandwagon": 49401, "\u0120refuted": 49402, "\u0120netted": 49403, "\u0120Naples": 49404, "\u0120Genie": 49405, "perors": 49406, "\u0120fielded": 49407, "\u0120dere": 49408, "\u0120Parables": 49409, "lees": 49410, "\u0120trout": 49411, "aspers": 49412, "\u0120nihil": 49413, "\u0120happiest": 49414, "\u0120floppy": 49415, "\u0120Loft": 49416, "\u0120Heard": 49417, "\u0120unison": 49418, "\u0120lug": 49419, "\u0120Redmond": 49420, "classic": 49421, "Supporters": 49422, "SHIP": 49423, "GMT": 49424, "\u0120fuelled": 49425, "\u00e7\u0132": 49426, "\u0120dd": 49427, "\u0120Eminem": 49428, "\u01201897": 49429, "NYSE": 49430, "\u0120secretaries": 49431, "\u0120FIA": 49432, "\u0120Canaveral": 49433, "Favorite": 49434, "\u0120pomp": 49435, "\u0120detainee": 49436, "ership": 49437, "aimon": 49438, "iour": 49439, "\u0120Apex": 49440, "\u0120plantations": 49441, "amia": 49442, "acion": 49443, "Rust": 49444, "\u0120towed": 49445, "\u0120Truly": 49446, "577": 49447, "\u0120sheltered": 49448, "rider": 49449, "Wo": 49450, "\u0120lair": 49451, "\u0120Intelligent": 49452, "improve": 49453, "matically": 49454, "\u0120etiquette": 49455, "adra": 49456, "allo": 49457, "\u0120Juno": 49458, "anything": 49459, "\u0120Struggle": 49460, "\u0120Predict": 49461, "\u0120Grimes": 49462, "\u0120AMERICA": 49463, "ctx": 49464, "\u0120Situation": 49465, "WOOD": 49466, "\u0120soluble": 49467, "meier": 49468, "\u0120intolerable": 49469, "angering": 49470, "\u0120uninterrupted": 49471, "\u0120tooltip": 49472, "\u0120interrogated": 49473, "\u0120gunned": 49474, "\u0120Sneak": 49475, "\u00e6\u0143\u00a6": 49476, "\u0120tether": 49477, "\u0120crumble": 49478, "Lens": 49479, "\u0120clustered": 49480, "\u0120Syl": 49481, "\u0120Hasan": 49482, "\u0120dystopian": 49483, "wana": 49484, "\u0120joystick": 49485, "\u0120Thib": 49486, "ammu": 49487, "Tomorrow": 49488, "546": 49489, "\u0120overcame": 49490, "\u0120minimized": 49491, "ceptor": 49492, "Runner": 49493, "ENGTH": 49494, "\u0120Brenda": 49495, "\u0120Achievements": 49496, "\u0120torches": 49497, "\u0120rapport": 49498, "\u0120Investigator": 49499, "\u0120Handling": 49500, "relation": 49501, "grey": 49502, "815": 49503, "\u0120kcal": 49504, "\u0120Commands": 49505, "dq": 49506, "\u0120curls": 49507, "\u0120bearer": 49508, "\u0120cynicism": 49509, "itri": 49510, "\u0120Useful": 49511, "Bee": 49512, "DCS": 49513, "\u0120abras": 49514, "Pract": 49515, "BILITIES": 49516, "712": 49517, "\u0120debugger": 49518, "\u0120debtor": 49519, "\u0120Lia": 49520, "\u0120Kers": 49521, "\u0120exacerbate": 49522, "\u0120Stacy": 49523, "\u0120Bland": 49524, "\u0120Scenes": 49525, "\u0120branching": 49526, "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 49527, "apeake": 49528, "\u0120salsa": 49529, "\u0120mishand": 49530, "\u0120Konami": 49531, "\u0120Nib": 49532, "\u0120anecdote": 49533, "\u0120agreeable": 49534, "\u00cf\u012b": 49535, "\u0120Nathaniel": 49536, "\u0120Heisman": 49537, "\u0120Beware": 49538, "\u01201886": 49539, "spective": 49540, "691": 49541, "522": 49542, "\u0120inhibits": 49543, "\u0120hashing": 49544, "\u01201889": 49545, "\u00e5\u00b0\u0128": 49546, "vich": 49547, "Pure": 49548, "\u0120solidly": 49549, "\u0120aspirin": 49550, "imaru": 49551, "\u0120streetcar": 49552, "\u0120UCS": 49553, "\u0120Judd": 49554, "\u0120flashbacks": 49555, "pins": 49556, "\u01201440": 49557, "\u0120UNHCR": 49558, "\u0120Symptoms": 49559, "TIT": 49560, "538": 49561, "Fra": 49562, "%);": 49563, "\u0120ooz": 49564, "\u0120curfew": 49565, "\u0120calmed": 49566, "\u0120participates": 49567, "TeX": 49568, "\u0120nonsensical": 49569, "\u0120fullback": 49570, "\u0120DeL": 49571, "monkey": 49572, "hari": 49573, "\u0120metabolites": 49574, "\u0120looted": 49575, "\u0120ALWAYS": 49576, "\u0120BCC": 49577, "Lt": 49578, "ochet": 49579, "Bone": 49580, "\u0120vetoed": 49581, "\u0120gcc": 49582, "\u0120CLICK": 49583, "\u01201888": 49584, "saf": 49585, "\u0120stiffness": 49586, "\u0120lowly": 49587, "\u0120Geh": 49588, "verson": 49589, "orset": 49590, "\u0120unforeseen": 49591, "\u0120anesthesia": 49592, "\u0120Optical": 49593, "\u0120reconstructed": 49594, "\u0120Tup": 49595, "shows": 49596, "NEWS": 49597, "\u0120Newspaper": 49598, "\u0120ASA": 49599, "tera": 49600, "Numbers": 49601, "\u0120inexplicable": 49602, "\u00d7\u0133": 49603, "\u0120hardness": 49604, "untarily": 49605, "\u0120Acer": 49606, "gradient": 49607, "ARDIS": 49608, "\u0120woodland": 49609, "\u0120metaphors": 49610, "\u0120Wembley": 49611, "\u0120Pavel": 49612, "philis": 49613, "\u0120rewriting": 49614, "\u0120perceptual": 49615, "\u01201070": 49616, "worms": 49617, "\u0120Downs": 49618, "\u0120unsurprisingly": 49619, "\u0120tagging": 49620, "flame": 49621, "\u0120litres": 49622, "\u0120bounces": 49623, "\u0120Babe": 49624, "shut": 49625, "\u0120overdoses": 49626, "\u0120Sheila": 49627, "\u0120Chau": 49628, "\u0120Bless": 49629, "Capture": 49630, "\u0120Significant": 49631, "\u0120Scion": 49632, "\u0120389": 49633, "\u0120McH": 49634, "\u0120Titanium": 49635, "\u0120Meal": 49636, "ameda": 49637, "agents": 49638, "aggressive": 49639, "Billy": 49640, "763": 49641, "\u0120Saying": 49642, "DERR": 49643, "itone": 49644, "Collins": 49645, "Bound": 49646, "\u0120bolted": 49647, "\u0120DMCA": 49648, "953": 49649, "\u0120uniqueness": 49650, "\u0120epigen": 49651, "unci": 49652, "antam": 49653, "\u0120reckoning": 49654, "chairs": 49655, "OGR": 49656, "\u0120Senegal": 49657, "\u01201862": 49658, "relevant": 49659, "\u0120\u00c2\u00af": 49660, "\u0120pharmacies": 49661, "\u0120Geral": 49662, "vier": 49663, "Yan": 49664, "ORPG": 49665, "\u0120rabid": 49666, "bending": 49667, "\u0120UNITED": 49668, "\u0120465": 49669, "Assembly": 49670, "\u0120weep": 49671, "\u0120behest": 49672, "\u0120Mothers": 49673, "\u0120Jace": 49674, "hid": 49675, "\u0120whirlwind": 49676, "\u0120UNIVERS": 49677, "\u0120utopian": 49678, "\u0120kidnap": 49679, "Philipp": 49680, "Kin": 49681, "893": 49682, "\u0120livestream": 49683, "\u0120MISS": 49684, "\u0120subversive": 49685, "\u0120Techniques": 49686, "\u0120JUSTICE": 49687, "\u0120BASE": 49688, "\u0120387": 49689, "\u0120assailants": 49690, "\u0120Hardcore": 49691, "\u0120sprinkled": 49692, "\u0120Pse": 49693, "\u00e9\u013c": 49694, "printed": 49695, "\u0120Hau": 49696, "ORGE": 49697, "\u0120TOUR": 49698, "\u0120laced": 49699, "\u0120itch": 49700, "Giving": 49701, "\u0120ported": 49702, "781": 49703, "////////////////////////////////": 49704, "breeding": 49705, "\u0120logger": 49706, "\u0120HOL": 49707, "innie": 49708, "Firstly": 49709, "\u0120embryonic": 49710, "\u0120delegated": 49711, "pai": 49712, "OIL": 49713, "\u0120centrally": 49714, "\u0120Rx": 49715, "\u0120Scouting": 49716, "Dutch": 49717, "\u0120hereditary": 49718, "\u0120Cruiser": 49719, "sat": 49720, "529": 49721, "\u0120Marriott": 49722, "othermal": 49723, "\u0120prohibitions": 49724, "Earn": 49725, "\u0120Stab": 49726, "\u0120Colleges": 49727, "\u0120Belief": 49728, "stretched": 49729, "\u0120LH": 49730, "\u0120EntityItem": 49731, "CIA": 49732, "\u0120unrem": 49733, "\u0120laureate": 49734, "\u0120denominations": 49735, "summary": 49736, "hler": 49737, "Spect": 49738, "\u0120Klaus": 49739, "\u0120Beans": 49740, "\u0120insur": 49741, "\u0120PAX": 49742, "\u0120fielder": 49743, "\u0120Vet": 49744, "\u0120Sparrow": 49745, "zie": 49746, "\u0120SQ": 49747, "\u0120Mondays": 49748, "\u0120Offline": 49749, "\u0120Lerner": 49750, "\u0120Extensions": 49751, "Ireland": 49752, "\u0120patronage": 49753, "\u0120contrasted": 49754, "\u0120Mania": 49755, "hirt": 49756, "Moscow": 49757, "\u0120condemns": 49758, "\u0120Ange": 49759, "\u0120composing": 49760, "\u0120Pepe": 49761, "\u0120Paddock": 49762, "\u0120heterogeneity": 49763, "\u0120ideologically": 49764, "\u0120fishes": 49765, "\u0120cursing": 49766, "\u0120Rutherford": 49767, "\u0120Floating": 49768, "\u0120Amelia": 49769, "Tea": 49770, "Synopsis": 49771, "\u0120stunts": 49772, "\u0120bead": 49773, "\u0120stocking": 49774, "\u0120MILL": 49775, "obook": 49776, "massive": 49777, "\\<": 49778, "\u0120hump": 49779, "\u0120Preferences": 49780, "EngineDebug": 49781, "geist": 49782, "\u0120Nieto": 49783, "omever": 49784, "ishy": 49785, "evaluate": 49786, "colonial": 49787, "Alternative": 49788, "\u0120GoPro": 49789, "\u0120Vortex": 49790, "\u0120NETWORK": 49791, "ansky": 49792, "Secure": 49793, "\u0120Thrust": 49794, "Snake": 49795, "\u0120parcels": 49796, "\u0120samurai": 49797, "\u0120actresses": 49798, "Nap": 49799, "MF": 49800, "iferation": 49801, "Beer": 49802, "523": 49803, "\u0120Ily": 49804, "ointment": 49805, "Ping": 49806, "\u0120striped": 49807, "\u0120Mellon": 49808, "ossession": 49809, "\u0120neutron": 49810, "endium": 49811, "\u0120aph": 49812, "\u0120Flavoring": 49813, "\u0120383": 49814, "\u0120responsiveness": 49815, "\u0120Jindal": 49816, "\u0120Hitchcock": 49817, "Denver": 49818, "\u0120DRAGON": 49819, "smanship": 49820, "\u0120Dupl": 49821, "\u0120sly": 49822, "\u0120webcam": 49823, "\u0120Twain": 49824, "\u0120Darling": 49825, "iliate": 49826, "consumer": 49827, "DIT": 49828, "\u0120namesake": 49829, "\u0120unorthodox": 49830, "\u0120funer": 49831, "\u0120PLoS": 49832, "\u0120CONTROL": 49833, "ozyg": 49834, "oglobin": 49835, "FACE": 49836, "ERG": 49837, "\u0120Dia": 49838, "\u0120Fiesta": 49839, "cele": 49840, "034": 49841, "\u0120enclave": 49842, "\u00e2\u0138\u00ac\u00e2\u0138\u00ac": 49843, "onement": 49844, "alist": 49845, "Mand": 49846, "\u0120homegrown": 49847, "\u0120Fancy": 49848, "\u0120conceptions": 49849, "\u0120Contains": 49850, "ureen": 49851, "\u0120reiterate": 49852, "\u0120meager": 49853, "\u0120installments": 49854, "Spawn": 49855, "627": 49856, "\u0120photoc": 49857, "\u0120Cabrera": 49858, "\u0120Rosenthal": 49859, "\u0120Lansing": 49860, "isner": 49861, "\u0120invests": 49862, "\u0120UFOs": 49863, "EXP": 49864, "Hardware": 49865, "\u0120tragically": 49866, "\u0120concedes": 49867, "ieft": 49868, "cham": 49869, "borgh": 49870, "\u0120Schr": 49871, "\u0120Melanie": 49872, "\u0120Hoy": 49873, "\u0120visitation": 49874, "\u0120idiosyncr": 49875, "\u0120fractions": 49876, "\u0120foreskin": 49877, "obos": 49878, "\u0120poaching": 49879, "\u0120VIEW": 49880, "\u0120stimulates": 49881, "\u0120Gork": 49882, "canon": 49883, "MIC": 49884, "\u0120Nemesis": 49885, "\u0120Indra": 49886, "\u0120DMV": 49887, "\u0120529": 49888, "\u0120inspecting": 49889, "\u0120grandma": 49890, "\u0120Whedon": 49891, "\u0120Shant": 49892, "\u0120Purg": 49893, "ikan": 49894, "\u0120Teg": 49895, "\u0120CLR": 49896, "zac": 49897, "Victoria": 49898, "\u0120Verify": 49899, "ionics": 49900, "\u0120partying": 49901, "\u0120Mou": 49902, "colour": 49903, "\u0120testimonies": 49904, "lations": 49905, "\u0120pressuring": 49906, "hiro": 49907, "acers": 49908, "\u0120fid": 49909, "angler": 49910, "\u0120CSI": 49911, "\u0120hereafter": 49912, "\u0120dissidents": 49913, "reporting": 49914, "iphany": 49915, "chev": 49916, "\u0120solitude": 49917, "\u0120lobe": 49918, "\u0120indis": 49919, "\u0120credential": 49920, "recent": 49921, "adult": 49922, "\u0120Nirvana": 49923, "\u0120Franchise": 49924, "Layer": 49925, "Hyp": 49926, "\u0120Berkshire": 49927, "\u0120wills": 49928, "tif": 49929, "\u0120totem": 49930, "\u0120Judah": 49931, "repair": 49932, "Instant": 49933, "548": 49934, "\u0120embassies": 49935, "\u0120bottleneck": 49936, "\u0120bount": 49937, "\u0120typew": 49938, "\u0120Alvin": 49939, "jing": 49940, "imilar": 49941, "Rush": 49942, "\u0120brim": 49943, "\u0120HELP": 49944, "Aim": 49945, "]'": 49946, "\u0120passively": 49947, "\u0120bounded": 49948, "\u0120Rated": 49949, "\u0120criminality": 49950, "\u0120biomark": 49951, "\u0120dispatcher": 49952, "\u0120Towards": 49953, "\u0120+++": 49954, "righteous": 49955, "frog": 49956, "\u0120Panc": 49957, "Carter": 49958, "032": 49959, "\u00e6\u00a9\u0141": 49960, "\u0120ultraviolet": 49961, "\u0120Licensed": 49962, "\u0120Tata": 49963, "\u0120Blessing": 49964, "\u0120GAM": 49965, "\u0120chemically": 49966, "\u0120Seaf": 49967, "\u0120RELE": 49968, "\u0120Mercenary": 49969, "capitalist": 49970, "\u0120formulations": 49971, "\u0120annihilation": 49972, "\u0120Verb": 49973, "\u0120Argon": 49974, "\u0120unloaded": 49975, "\u0120morphed": 49976, "\u0120conquering": 49977, "backer": 49978, "IELD": 49979, "\u0120thefts": 49980, "\u0120frontrunner": 49981, "\u0120Royale": 49982, "\u0120Fundamental": 49983, "elight": 49984, "Chip": 49985, "necessary": 49986, "ayn": 49987, "\u0120Slip": 49988, "\u0120448": 49989, "cerned": 49990, "Pause": 49991, "\u0120shockingly": 49992, "\u0120ABV": 49993, "\u0120composure": 49994, "733": 49995, "\u0120Motorsport": 49996, "ahime": 49997, "Murray": 49998, "Mach": 49999, "\u0120grids": 50000, "\u0120debian": 50001, "\u0120furthermore": 50002, "\u0120dexterity": 50003, "\u0120Collections": 50004, "oslov": 50005, "ilage": 50006, "bj": 50007, "\u0120Monteneg": 50008, "\u0120strutConnector": 50009, "\u0120massacres": 50010, "\u0120briefs": 50011, "fetched": 50012, "uvian": 50013, "olition": 50014, "Failure": 50015, "emonic": 50016, "\u0120flared": 50017, "\u0120claimant": 50018, "\u0120cures": 50019, "\u0120giveaways": 50020, "\u0120Substance": 50021, "alions": 50022, "\u0120cringe": 50023, "\u0120Kul": 50024, "\u0120aristocracy": 50025, "\u0120Ulster": 50026, "olated": 50027, "housing": 50028, "\u0120MIS": 50029, "\u0120glared": 50030, "\u0120Wilhelm": 50031, "needs": 50032, "lambda": 50033, "builders": 50034, "\u0120VIS": 50035, "\u0120radiator": 50036, "\u0120Ghostbusters": 50037, "\u0120436": 50038, "actual": 50039, "\u0120herds": 50040, "\u00c3\u00a7a": 50041, "watching": 50042, "\u0120countering": 50043, "Charge": 50044, "\u0120charred": 50045, "\u0120warheads": 50046, "\u0120iodine": 50047, "\u0120Macy": 50048, "041": 50049, "\u0120departures": 50050, "\u0120Sins": 50051, "\u0120dyed": 50052, "\u0120Concepts": 50053, "gado": 50054, "713": 50055, "\u0120quotations": 50056, "\u0120gist": 50057, "\u0120Christy": 50058, "\u0120antigen": 50059, "\u0120Hemp": 50060, "\u0120Drawn": 50061, "\u0120Barg": 50062, "ezvous": 50063, "\u0120paternity": 50064, "\u0120ardu": 50065, "\u0120Anchorage": 50066, "\u0120Rik": 50067, "\u0120overloaded": 50068, "\u0120Username": 50069, "\u0120Tammy": 50070, "\u0120Nau": 50071, "\u0120Cellular": 50072, "\u0120waning": 50073, "\u0120rodent": 50074, "\u0120Worcester": 50075, "ilts": 50076, "\u0120Tad": 50077, "\u0120dwellings": 50078, "\u0120bullish": 50079, "431": 50080, "\u0120retaliate": 50081, "\u0120migraine": 50082, "\u0120Chevron": 50083, "CHECK": 50084, "\u0120donkey": 50085, "crim": 50086, "SPA": 50087, "\u0120Analog": 50088, "\u0120marquee": 50089, "\u0120Haas": 50090, "Bir": 50091, "\u0120GDDR": 50092, "\u0120Downloads": 50093, "\u0120willpower": 50094, "\u0120Forth": 50095, "\u0120Recorded": 50096, "\u0120impossibility": 50097, "\u0120Logged": 50098, "\u0120Franks": 50099, "\u0120Ratt": 50100, "initions": 50101, "\u0120cleaners": 50102, "\u0120sorely": 50103, "\u0120flickering": 50104, "\u0120Examination": 50105, "catching": 50106, "alloween": 50107, "Msg": 50108, "\u0120dunno": 50109, "Fa": 50110, "\u0120dysph": 50111, "crazy": 50112, ".''.": 50113, "\u0120mainline": 50114, "\u0120cs": 50115, "\u0120ptr": 50116, "\u0120Wally": 50117, "igun": 50118, "951": 50119, "\u0120Bigfoot": 50120, "fights": 50121, "\u0120retrieving": 50122, "Jr": 50123, "\u0120duplication": 50124, "\u0120Explan": 50125, "\u0120relational": 50126, "\u0120quaint": 50127, "\u0120biscuits": 50128, "\u0120ado": 50129, "\u0120shudder": 50130, "\u0120antidote": 50131, "blooded": 50132, "ksh": 50133, "\u0120sauces": 50134, "\u0120reinvest": 50135, "\u0120dispensary": 50136, "\u0120Diver": 50137, "\u01209000": 50138, "student": 50139, "\u0120insepar": 50140, "escap": 50141, "\u0120toddlers": 50142, "\u0120GPIO": 50143, "\u0120Assignment": 50144, "headers": 50145, "\u0120lackluster": 50146, "\u0120aback": 50147, "956": 50148, "\u0120toolbar": 50149, "745": 50150, "\u0120oust": 50151, "\u0120contemplation": 50152, "\u0120PRESIDENT": 50153, "\u0120458": 50154, "======": 50155, "\u0120guaranteeing": 50156, "\u0120Heist": 50157, "\u0120Cannes": 50158, "\u013b\u00bd": 50159, "\u0120collaborator": 50160, "\u0120Amp": 50161, "\u0120gou": 50162, "\u0120SHALL": 50163, "stories": 50164, "783": 50165, "\u0120mobilized": 50166, "\u0120brood": 50167, "\u0120LU": 50168, "\u0120\u00f0\u0141\u0133": 50169, "\u0120refin": 50170, "\u0120Anthropology": 50171, "vind": 50172, "illi": 50173, "\u0120warranties": 50174, "\u0120Babel": 50175, "\u0120swath": 50176, "\u0120caches": 50177, "\u0120antagonists": 50178, "artifacts": 50179, "\u0120hotly": 50180, "\u0120Starts": 50181, "\u0120G\u00c3\u00b6": 50182, "zag": 50183, "!!!!!": 50184, "\u0120scourge": 50185, "\u0120conspiring": 50186, "ruits": 50187, "reverse": 50188, "\u0120Sheen": 50189, "\u0120Jesuit": 50190, "\u0120Giovanni": 50191, "adies": 50192, "\u0120buttocks": 50193, "earcher": 50194, "acan": 50195, "\u0120volleyball": 50196, "\u0120shrouded": 50197, "\u0120scoreboard": 50198, "bats": 50199, "\u0120IPM": 50200, "\u0120asses": 50201, "\u0120deregulation": 50202, "\u0120Telegram": 50203, "\u0120Reboot": 50204, "\u01207000": 50205, "\u0120Canary": 50206, "\u0120kernels": 50207, "\u0120Fran\u00c3\u00a7ois": 50208, "\u0120Duff": 50209, "\u0120Pon": 50210, "\u0120Leica": 50211, "\u0120Garmin": 50212, "\u0120orphans": 50213, "\u0120Claudia": 50214, "\u0120calendars": 50215, "\u0120Leilan": 50216, "ento": 50217, "Rocket": 50218, "\u0120brunch": 50219, "\u0120Hawking": 50220, "ainers": 50221, "\u0120sensibilities": 50222, "\u0120kW": 50223, "\u0120Kand": 50224, "\u0120reclaimed": 50225, "\u0120interestingly": 50226, "\u00d7\u00a9": 50227, "romy": 50228, "JM": 50229, "\u0120Enhancement": 50230, "bush": 50231, "Skip": 50232, "\u0120rappers": 50233, "\u0120gazing": 50234, "pedia": 50235, "athlon": 50236, "Revolution": 50237, "\u0120snipers": 50238, "\u0120reverted": 50239, "\u0120conglomerate": 50240, "Terry": 50241, "794": 50242, "\u0120harsher": 50243, "\u0120desolate": 50244, "\u0120Hitman": 50245, "Commission": 50246, "\u0120(/": 50247, "\u00e2\u0122\u00a6.\"": 50248, "Compar": 50249, "\u0120amplification": 50250, "ominated": 50251, "\u0120regress": 50252, "\u0120Collider": 50253, "\u0120informants": 50254, "\u0120gazed": 50255, "<|endoftext|>": 50256} \ No newline at end of file diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/video2world.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/video2world.py new file mode 100755 index 00000000..2f9ad232 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/nemo/post_training/video2world.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +from huggingface_hub import snapshot_download +from nemo.collections import llm +from nemo.collections.diffusion.models.model import ( + DiT7BCameraCtrlConfig, + DiT7BVideo2WorldConfig, + DiT14BVideo2WorldConfig, +) +from nemo.collections.diffusion.train import ( + finetune_7b_action_control, + finetune_14b_action_control, + pretrain, + videofolder_cameractrldatamodule, + videofolder_datamodule, +) +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_7b_video2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(DiT7BVideo2WorldConfig) + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.ckpt_load_strictness = "log_all" + + # FSDP + # recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + # recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = False + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.model.config.use_cpu_initialization = True + + # Activation Checkpointing + recipe.model.config.recompute_granularity = "full" + recipe.model.config.recompute_method = "uniform" + recipe.model.config.recompute_num_layers = 1 + + # Data setup + recipe.data = videofolder_datamodule() + recipe.data.path = "" # path to folder with processed dataset, can pass in via nemo-run CLI + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_diffusion_7b_video2world_finetune" + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_7b_cameractrl_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(DiT7BCameraCtrlConfig) + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.ckpt_load_strictness = False + + # FSDP + recipe.trainer.strategy.ddp.check_for_nan_in_grad = True + recipe.trainer.strategy.ddp.grad_reduce_in_fp32 = True + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.trainer.strategy.ddp.overlap_param_gather = False + + # Activation Checkpointing + recipe.model.config.recompute_granularity = "full" + recipe.model.config.recompute_method = "uniform" + recipe.model.config.recompute_num_layers = 1 + + # Data setup + recipe.data = videofolder_cameractrldatamodule() + recipe.data.path = "" # path to folder with processed dataset, can pass in via nemo-run CLI + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_diffusion_7b_cameractrl_finetune" + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_14b_video2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(DiT14BVideo2WorldConfig) + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.ckpt_load_strictness = "log_all" + + # FSDP + # recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + # recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = False + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.model.config.use_cpu_initialization = True + + # Activation Checkpointing + recipe.model.config.recompute_granularity = "full" + recipe.model.config.recompute_method = "uniform" + recipe.model.config.recompute_num_layers = 1 + + # Data setup + recipe.data = videofolder_datamodule() + recipe.data.path = "" # path to folder with processed dataset + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-14B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_diffusion_14b_video2world_finetune" + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_7b_video2world_action_ctrl_finetune() -> run.Partial: + # Associated with the https://gitlab-master.nvidia.com/dl/nemo/nemo-vfm. + recipe = finetune_7b_action_control() + + # Load Pre-Trained Checkpoint. + recipe.resume.restore_config = run.Config( + RestoreConfig, load_model_state=True, load_optim_state=False, load_artifacts=False + ) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_diffusion_14b_video2world_action_ctrl_finetune() -> run.Partial: + # Associated with the https://gitlab-master.nvidia.com/dl/nemo/nemo-vfm. + recipe = finetune_14b_action_control() + + # Load Pre-Trained Checkpoint. + recipe.resume.restore_config = run.Config( + RestoreConfig, load_model_state=True, load_optim_state=False, load_artifacts=False + ) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-14B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=cosmos_diffusion_7b_video2world_finetune) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit.py new file mode 100644 index 00000000..8160617d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit.py @@ -0,0 +1,521 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +A general implementation of adaln-modulated VIT-like~(DiT) transformer for video processing. +""" + +from typing import List, Optional, Tuple + +import torch +from cosmos1.models.diffusion.conditioner import DataType +from cosmos1.models.diffusion.module.attention import get_normalization +from cosmos1.models.diffusion.module.blocks import ( + FinalLayer, + GeneralDITTransformerBlock, + PatchEmbed, + TimestepEmbedding, + Timesteps, +) +from cosmos1.models.diffusion.module.position_embedding import LearnablePosEmbAxis, VideoRopePosition3DEmb +from cosmos1.utils import log +from einops import rearrange +from torch import nn +from torchvision import transforms + + +class GeneralDIT(nn.Module): + """ + A general implementation of adaln-modulated VIT-like~(DiT) transformer for video processing. + + Args: + max_img_h (int): Maximum height of the input images. + max_img_w (int): Maximum width of the input images. + max_frames (int): Maximum number of frames in the video sequence. + in_channels (int): Number of input channels (e.g., RGB channels for color images). + out_channels (int): Number of output channels. + patch_spatial (tuple): Spatial resolution of patches for input processing. + patch_temporal (int): Temporal resolution of patches for input processing. + concat_padding_mask (bool): If True, includes a mask channel in the input to handle padding. + block_config (str): Configuration of the transformer block. See Notes for supported block types. + model_channels (int): Base number of channels used throughout the model. + num_blocks (int): Number of transformer blocks. + num_heads (int): Number of heads in the multi-head attention layers. + mlp_ratio (float): Expansion ratio for MLP blocks. + block_x_format (str): Format of input tensor for transformer blocks ('BTHWD' or 'THWBD'). + crossattn_emb_channels (int): Number of embedding channels for cross-attention. + use_cross_attn_mask (bool): Whether to use mask in cross-attention. + pos_emb_cls (str): Type of positional embeddings. + pos_emb_learnable (bool): Whether positional embeddings are learnable. + pos_emb_interpolation (str): Method for interpolating positional embeddings. + affline_emb_norm (bool): Whether to normalize affine embeddings. + use_adaln_lora (bool): Whether to use AdaLN-LoRA. + adaln_lora_dim (int): Dimension for AdaLN-LoRA. + rope_h_extrapolation_ratio (float): Height extrapolation ratio for RoPE. + rope_w_extrapolation_ratio (float): Width extrapolation ratio for RoPE. + rope_t_extrapolation_ratio (float): Temporal extrapolation ratio for RoPE. + extra_per_block_abs_pos_emb (bool): Whether to use extra per-block absolute positional embeddings. + extra_per_block_abs_pos_emb_type (str): Type of extra per-block positional embeddings. + extra_h_extrapolation_ratio (float): Height extrapolation ratio for extra embeddings. + extra_w_extrapolation_ratio (float): Width extrapolation ratio for extra embeddings. + extra_t_extrapolation_ratio (float): Temporal extrapolation ratio for extra embeddings. + + Notes: + Supported block types in block_config: + * cross_attn, ca: Cross attention + * full_attn: Full attention on all flattened tokens + * mlp, ff: Feed forward block + """ + + def __init__( + self, + max_img_h: int, + max_img_w: int, + max_frames: int, + in_channels: int, + out_channels: int, + patch_spatial: tuple, + patch_temporal: int, + concat_padding_mask: bool = True, + # attention settings + block_config: str = "FA-CA-MLP", + model_channels: int = 768, + num_blocks: int = 10, + num_heads: int = 16, + mlp_ratio: float = 4.0, + block_x_format: str = "BTHWD", + # cross attention settings + crossattn_emb_channels: int = 1024, + use_cross_attn_mask: bool = False, + # positional embedding settings + pos_emb_cls: str = "sincos", + pos_emb_learnable: bool = False, + pos_emb_interpolation: str = "crop", + affline_emb_norm: bool = False, # whether or not to normalize the affine embedding + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + rope_h_extrapolation_ratio: float = 1.0, + rope_w_extrapolation_ratio: float = 1.0, + rope_t_extrapolation_ratio: float = 1.0, + extra_per_block_abs_pos_emb: bool = False, + extra_per_block_abs_pos_emb_type: str = "sincos", + extra_h_extrapolation_ratio: float = 1.0, + extra_w_extrapolation_ratio: float = 1.0, + extra_t_extrapolation_ratio: float = 1.0, + ) -> None: + super().__init__() + self.max_img_h = max_img_h + self.max_img_w = max_img_w + self.max_frames = max_frames + self.in_channels = in_channels + self.out_channels = out_channels + self.patch_spatial = patch_spatial + self.patch_temporal = patch_temporal + self.num_heads = num_heads + self.num_blocks = num_blocks + self.model_channels = model_channels + self.use_cross_attn_mask = use_cross_attn_mask + self.concat_padding_mask = concat_padding_mask + # positional embedding settings + self.pos_emb_cls = pos_emb_cls + self.pos_emb_learnable = pos_emb_learnable + self.pos_emb_interpolation = pos_emb_interpolation + self.affline_emb_norm = affline_emb_norm + self.rope_h_extrapolation_ratio = rope_h_extrapolation_ratio + self.rope_w_extrapolation_ratio = rope_w_extrapolation_ratio + self.rope_t_extrapolation_ratio = rope_t_extrapolation_ratio + self.extra_per_block_abs_pos_emb = extra_per_block_abs_pos_emb + self.extra_per_block_abs_pos_emb_type = extra_per_block_abs_pos_emb_type.lower() + self.extra_h_extrapolation_ratio = extra_h_extrapolation_ratio + self.extra_w_extrapolation_ratio = extra_w_extrapolation_ratio + self.extra_t_extrapolation_ratio = extra_t_extrapolation_ratio + + self.build_patch_embed() + self.build_pos_embed() + self.block_x_format = block_x_format + self.use_adaln_lora = use_adaln_lora + self.adaln_lora_dim = adaln_lora_dim + self.t_embedder = nn.Sequential( + Timesteps(model_channels), + TimestepEmbedding(model_channels, model_channels, use_adaln_lora=use_adaln_lora), + ) + + self.blocks = nn.ModuleDict() + + for idx in range(num_blocks): + self.blocks[f"block{idx}"] = GeneralDITTransformerBlock( + x_dim=model_channels, + context_dim=crossattn_emb_channels, + num_heads=num_heads, + block_config=block_config, + mlp_ratio=mlp_ratio, + x_format=self.block_x_format, + use_adaln_lora=use_adaln_lora, + adaln_lora_dim=adaln_lora_dim, + ) + + self.build_decode_head() + if self.affline_emb_norm: + log.debug("Building affine embedding normalization layer") + self.affline_norm = get_normalization("R", model_channels) + else: + self.affline_norm = nn.Identity() + self.initialize_weights() + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + + self.apply(_basic_init) + + # Initialize timestep embedding + nn.init.normal_(self.t_embedder[1].linear_1.weight, std=0.02) + if self.t_embedder[1].linear_1.bias is not None: + nn.init.constant_(self.t_embedder[1].linear_1.bias, 0) + nn.init.normal_(self.t_embedder[1].linear_2.weight, std=0.02) + if self.t_embedder[1].linear_2.bias is not None: + nn.init.constant_(self.t_embedder[1].linear_2.bias, 0) + + # Zero-out adaLN modulation layers in DiT blocks: + for transformer_block in self.blocks.values(): + for block in transformer_block.blocks: + nn.init.constant_(block.adaLN_modulation[-1].weight, 0) + if block.adaLN_modulation[-1].bias is not None: + nn.init.constant_(block.adaLN_modulation[-1].bias, 0) + + def build_decode_head(self): + self.final_layer = FinalLayer( + hidden_size=self.model_channels, + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + out_channels=self.out_channels, + use_adaln_lora=self.use_adaln_lora, + adaln_lora_dim=self.adaln_lora_dim, + ) + + def build_patch_embed(self): + ( + concat_padding_mask, + in_channels, + patch_spatial, + patch_temporal, + model_channels, + ) = ( + self.concat_padding_mask, + self.in_channels, + self.patch_spatial, + self.patch_temporal, + self.model_channels, + ) + in_channels = in_channels + 1 if concat_padding_mask else in_channels + self.x_embedder = PatchEmbed( + spatial_patch_size=patch_spatial, + temporal_patch_size=patch_temporal, + in_channels=in_channels, + out_channels=model_channels, + bias=False, + ) + + def build_pos_embed(self): + if self.pos_emb_cls == "rope3d": + cls_type = VideoRopePosition3DEmb + else: + raise ValueError(f"Unknown pos_emb_cls {self.pos_emb_cls}") + + log.debug(f"Building positional embedding with {self.pos_emb_cls} class, impl {cls_type}") + kwargs = dict( + model_channels=self.model_channels, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + is_learnable=self.pos_emb_learnable, + interpolation=self.pos_emb_interpolation, + head_dim=self.model_channels // self.num_heads, + h_extrapolation_ratio=self.rope_h_extrapolation_ratio, + w_extrapolation_ratio=self.rope_w_extrapolation_ratio, + t_extrapolation_ratio=self.rope_t_extrapolation_ratio, + ) + self.pos_embedder = cls_type( + **kwargs, + ) + + if self.extra_per_block_abs_pos_emb: + assert self.extra_per_block_abs_pos_emb_type in [ + "learnable", + ], f"Unknown extra_per_block_abs_pos_emb_type {self.extra_per_block_abs_pos_emb_type}" + kwargs["h_extrapolation_ratio"] = self.extra_h_extrapolation_ratio + kwargs["w_extrapolation_ratio"] = self.extra_w_extrapolation_ratio + kwargs["t_extrapolation_ratio"] = self.extra_t_extrapolation_ratio + self.extra_pos_embedder = LearnablePosEmbAxis( + **kwargs, + ) + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Prepares an embedded sequence tensor by applying positional embeddings and handling padding masks. + + Args: + x_B_C_T_H_W (torch.Tensor): video + fps (Optional[torch.Tensor]): Frames per second tensor to be used for positional embedding when required. + If None, a default value (`self.base_fps`) will be used. + padding_mask (Optional[torch.Tensor]): current it is not used + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - A tensor of shape (B, T, H, W, D) with the embedded sequence. + - An optional positional embedding tensor, returned only if the positional embedding class + (`self.pos_emb_cls`) includes 'rope'. Otherwise, None. + + Notes: + - If `self.concat_padding_mask` is True, a padding mask channel is concatenated to the input tensor. + - The method of applying positional embeddings depends on the value of `self.pos_emb_cls`. + - If 'rope' is in `self.pos_emb_cls` (case insensitive), the positional embeddings are generated using + the `self.pos_embedder` with the shape [T, H, W]. + - If "fps_aware" is in `self.pos_emb_cls`, the positional embeddings are generated using the + `self.pos_embedder` with the fps tensor. + - Otherwise, the positional embeddings are generated without considering fps. + """ + if self.concat_padding_mask: + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + + return x_B_T_H_W_D, None, extra_pos_emb + + def decoder_head( + self, + x_B_T_H_W_D: torch.Tensor, + emb_B_D: torch.Tensor, + crossattn_emb: torch.Tensor, + origin_shape: Tuple[int, int, int, int, int], # [B, C, T, H, W] + crossattn_mask: Optional[torch.Tensor] = None, + adaln_lora_B_3D: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + del crossattn_emb, crossattn_mask + B, C, T_before_patchify, H_before_patchify, W_before_patchify = origin_shape + x_BT_HW_D = rearrange(x_B_T_H_W_D, "B T H W D -> (B T) (H W) D") + x_BT_HW_D = self.final_layer(x_BT_HW_D, emb_B_D, adaln_lora_B_3D=adaln_lora_B_3D) + # This is to ensure x_BT_HW_D has the correct shape because + # when we merge T, H, W into one dimension, x_BT_HW_D has shape (B * T * H * W, 1*1, D). + x_BT_HW_D = x_BT_HW_D.view( + B * T_before_patchify // self.patch_temporal, + H_before_patchify // self.patch_spatial * W_before_patchify // self.patch_spatial, + -1, + ) + x_B_D_T_H_W = rearrange( + x_BT_HW_D, + "(B T) (H W) (p1 p2 t C) -> B C (T t) (H p1) (W p2)", + p1=self.patch_spatial, + p2=self.patch_spatial, + H=H_before_patchify // self.patch_spatial, + W=W_before_patchify // self.patch_spatial, + t=self.patch_temporal, + B=B, + ) + return x_B_D_T_H_W + + def forward_before_blocks( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + fps: Optional[torch.Tensor] = None, + image_size: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + Args: + x: (B, C, T, H, W) tensor of spatial-temp inputs + timesteps: (B, ) tensor of timesteps + crossattn_emb: (B, N, D) tensor of cross-attention embeddings + crossattn_mask: (B, N) tensor of cross-attention masks + """ + del kwargs + assert isinstance(data_type, DataType), ( + f"Expected DataType, got {type(data_type)}. We need discuss this flag later." + ) + original_shape = x.shape + x_B_T_H_W_D, rope_emb_L_1_1_D, extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = self.prepare_embedded_sequence( + x, + fps=fps, + padding_mask=padding_mask, + latent_condition=latent_condition, + latent_condition_sigma=latent_condition_sigma, + ) + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if scalar_feature is not None: + raise NotImplementedError("Scalar feature is not implemented yet.") + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + if self.use_cross_attn_mask: + crossattn_mask = crossattn_mask[:, None, None, :].to(dtype=torch.bool) # [B, 1, 1, length] + else: + crossattn_mask = None + + if self.blocks["block0"].x_format == "THWBD": + x = rearrange(x_B_T_H_W_D, "B T H W D -> T H W B D") + if extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D is not None: + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = rearrange( + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, "B T H W D -> T H W B D" + ) + crossattn_emb = rearrange(crossattn_emb, "B M D -> M B D") + + if crossattn_mask: + crossattn_mask = rearrange(crossattn_mask, "B M -> M B") + + elif self.blocks["block0"].x_format == "BTHWD": + x = x_B_T_H_W_D + else: + raise ValueError(f"Unknown x_format {self.blocks[0].x_format}") + output = { + "x": x, + "affline_emb_B_D": affline_emb_B_D, + "crossattn_emb": crossattn_emb, + "crossattn_mask": crossattn_mask, + "rope_emb_L_1_1_D": rope_emb_L_1_1_D, + "adaln_lora_B_3D": adaln_lora_B_3D, + "original_shape": original_shape, + "extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D": extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, + } + return output + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + fps: Optional[torch.Tensor] = None, + image_size: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + condition_video_augment_sigma: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor | List[torch.Tensor] | Tuple[torch.Tensor, List[torch.Tensor]]: + """ + Args: + x: (B, C, T, H, W) tensor of spatial-temp inputs + timesteps: (B, ) tensor of timesteps + crossattn_emb: (B, N, D) tensor of cross-attention embeddings + crossattn_mask: (B, N) tensor of cross-attention masks + condition_video_augment_sigma: (B,) used in lvg(long video generation), we add noise with this sigma to + augment condition input, the lvg model will condition on the condition_video_augment_sigma value; + we need forward_before_blocks pass to the forward_before_blocks function. + """ + + inputs = self.forward_before_blocks( + x=x, + timesteps=timesteps, + crossattn_emb=crossattn_emb, + crossattn_mask=crossattn_mask, + fps=fps, + image_size=image_size, + padding_mask=padding_mask, + scalar_feature=scalar_feature, + data_type=data_type, + latent_condition=latent_condition, + latent_condition_sigma=latent_condition_sigma, + condition_video_augment_sigma=condition_video_augment_sigma, + **kwargs, + ) + x, affline_emb_B_D, crossattn_emb, crossattn_mask, rope_emb_L_1_1_D, adaln_lora_B_3D, original_shape = ( + inputs["x"], + inputs["affline_emb_B_D"], + inputs["crossattn_emb"], + inputs["crossattn_mask"], + inputs["rope_emb_L_1_1_D"], + inputs["adaln_lora_B_3D"], + inputs["original_shape"], + ) + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = inputs["extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D"] + if extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D is not None: + assert x.shape == extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D.shape, ( + f"{x.shape} != {extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D.shape} {original_shape}" + ) + + for _, block in self.blocks.items(): + assert self.blocks["block0"].x_format == block.x_format, ( + f"First block has x_format {self.blocks[0].x_format}, got {block.x_format}" + ) + + x = block( + x, + affline_emb_B_D, + crossattn_emb, + crossattn_mask, + rope_emb_L_1_1_D=rope_emb_L_1_1_D, + adaln_lora_B_3D=adaln_lora_B_3D, + extra_per_block_pos_emb=extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, + ) + + x_B_T_H_W_D = rearrange(x, "T H W B D -> B T H W D") + + x_B_D_T_H_W = self.decoder_head( + x_B_T_H_W_D=x_B_T_H_W_D, + emb_B_D=affline_emb_B_D, + crossattn_emb=None, + origin_shape=original_shape, + crossattn_mask=None, + adaln_lora_B_3D=adaln_lora_B_3D, + ) + + return x_B_D_T_H_W diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit_multi_camera.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit_multi_camera.py new file mode 100644 index 00000000..d8309367 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit_multi_camera.py @@ -0,0 +1,458 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional, Tuple + +import torch +from cosmos1.models.diffusion.conditioner import DataType +from cosmos1.models.diffusion.module.blocks import GeneralDITTransformerBlock, PatchEmbed +from cosmos1.models.diffusion.module.position_embedding import ( + MultiCameraSinCosPosEmbAxis, + MultiCameraVideoRopePosition3DEmb, +) +from cosmos1.models.diffusion.networks.general_dit import GeneralDIT +from cosmos1.utils import log +from einops import rearrange +from torch import nn +from torchvision import transforms + + +class MultiCameraGeneralDIT(GeneralDIT): + def __init__( + self, + max_img_h: int, + max_img_w: int, + max_frames: int, + in_channels: int, + out_channels: int, + patch_spatial: tuple, + patch_temporal: int, + concat_padding_mask: bool = True, + # attention settings + block_config: str = "FA-CA-MLP", + model_channels: int = 768, + num_blocks: int = 10, + num_heads: int = 16, + mlp_ratio: float = 4.0, + block_x_format: str = "BTHWD", + # cross attention settings + crossattn_emb_channels: int = 1024, + use_cross_attn_mask: bool = False, + # positional embedding settings + pos_emb_cls: str = "sincos", + pos_emb_learnable: bool = False, + pos_emb_interpolation: str = "crop", + affline_emb_norm: bool = False, # whether or not to normalize the affine embedding + use_adaln_lora: bool = False, + adaln_lora_dim: int = 256, + rope_h_extrapolation_ratio: float = 1.0, + rope_w_extrapolation_ratio: float = 1.0, + rope_t_extrapolation_ratio: float = 1.0, + extra_per_block_abs_pos_emb: bool = False, + extra_per_block_abs_pos_emb_type: str = "sincos", + extra_h_extrapolation_ratio: float = 1.0, + extra_w_extrapolation_ratio: float = 1.0, + extra_t_extrapolation_ratio: float = 1.0, + n_cameras: int = 3, + camera_condition_dim: int = 3, + traj_condition_dim: int = 0, + concat_camera_embedding: bool = True, + concat_traj_embedding: bool = False, + add_repeat_frame_embedding: bool = False, + ): + self.n_cameras = n_cameras + self.camera_condition_dim = camera_condition_dim + self.concat_camera_embedding = concat_camera_embedding + self.traj_condition_dim = traj_condition_dim + self.concat_traj_embedding = concat_traj_embedding + self.add_repeat_frame_embedding = add_repeat_frame_embedding + super().__init__( + max_img_h, + max_img_w, + max_frames, + in_channels, + out_channels, + patch_spatial, + patch_temporal, + concat_padding_mask, + block_config, + model_channels, + num_blocks, + num_heads, + mlp_ratio, + block_x_format, + crossattn_emb_channels, + use_cross_attn_mask, + pos_emb_cls, + pos_emb_learnable, + pos_emb_interpolation, + affline_emb_norm, # whether or not to normalize the affine embedding + use_adaln_lora, + adaln_lora_dim, + rope_h_extrapolation_ratio, + rope_w_extrapolation_ratio, + rope_t_extrapolation_ratio, + extra_per_block_abs_pos_emb, + extra_per_block_abs_pos_emb_type, + extra_h_extrapolation_ratio, + extra_w_extrapolation_ratio, + extra_t_extrapolation_ratio, + ) + # reinit self.blocks + del self.blocks + self.blocks = nn.ModuleDict() + for idx in range(self.num_blocks): + self.blocks[f"block{idx}"] = GeneralDITTransformerBlock( + x_dim=model_channels, + context_dim=crossattn_emb_channels, + num_heads=num_heads, + block_config=block_config, + mlp_ratio=mlp_ratio, + x_format=self.block_x_format, + use_adaln_lora=use_adaln_lora, + adaln_lora_dim=adaln_lora_dim, + n_cameras=self.n_cameras, + ) + self.view_embeddings = nn.Embedding(n_cameras, camera_condition_dim) # Learnable embedding layer + if self.concat_traj_embedding: + self.traj_embeddings = nn.Linear(192, self.traj_condition_dim) # Learnable embedding layer + if self.add_repeat_frame_embedding: + self.repeat_frame_embedding = nn.Linear(1, camera_condition_dim) # Learnable embedding layer + + self.initialize_weights() + + def build_patch_embed(self): + ( + concat_padding_mask, + in_channels, + patch_spatial, + patch_temporal, + model_channels, + camera_condition_dim, + traj_condition_dim, + ) = ( + self.concat_padding_mask, + self.in_channels, + self.patch_spatial, + self.patch_temporal, + self.model_channels, + self.camera_condition_dim, + self.traj_condition_dim, + ) + if self.concat_camera_embedding: + in_channels = in_channels + camera_condition_dim if camera_condition_dim > 0 else in_channels + + if self.concat_traj_embedding: + in_channels = in_channels + traj_condition_dim if traj_condition_dim > 0 else in_channels + + in_channels = in_channels + 1 if concat_padding_mask else in_channels + + self.x_embedder = PatchEmbed( + spatial_patch_size=patch_spatial, + temporal_patch_size=patch_temporal, + in_channels=in_channels, + out_channels=model_channels, + bias=False, + ) + + def build_pos_embed(self): + if self.pos_emb_cls == "rope3d": + cls_type = MultiCameraVideoRopePosition3DEmb + else: + raise ValueError(f"Unknown pos_emb_cls {self.pos_emb_cls}") + + log.critical(f"Building positional embedding with {self.pos_emb_cls} class, impl {cls_type}") + kwargs = dict( + model_channels=self.model_channels, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + max_fps=30, + min_fps=1, + is_learnable=self.pos_emb_learnable, + interpolation=self.pos_emb_interpolation, + head_dim=self.model_channels // self.num_heads, + h_extrapolation_ratio=self.rope_h_extrapolation_ratio, + w_extrapolation_ratio=self.rope_w_extrapolation_ratio, + t_extrapolation_ratio=self.rope_t_extrapolation_ratio, + n_cameras=self.n_cameras, + ) + self.pos_embedder = cls_type( + **kwargs, + ) + + if self.extra_per_block_abs_pos_emb: + kwargs["h_extrapolation_ratio"] = self.extra_h_extrapolation_ratio + kwargs["w_extrapolation_ratio"] = self.extra_w_extrapolation_ratio + kwargs["t_extrapolation_ratio"] = self.extra_t_extrapolation_ratio + self.extra_pos_embedder = MultiCameraSinCosPosEmbAxis( + **kwargs, + ) + + def forward_before_blocks( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + fps: Optional[torch.Tensor] = None, + image_size: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + Args: + x: (B, C, T, H, W) tensor of spatial-temp inputs + timesteps: (B, ) tensor of timesteps + crossattn_emb: (B, N, D) tensor of cross-attention embeddings + crossattn_mask: (B, N) tensor of cross-attention masks + """ + trajectory = kwargs.get("trajectory", None) + frame_repeat = kwargs.get("frame_repeat", None) + + del kwargs + assert isinstance(data_type, DataType), ( + f"Expected DataType, got {type(data_type)}. We need discuss this flag later." + ) + original_shape = x.shape + x_B_T_H_W_D, rope_emb_L_1_1_D, extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = self.prepare_embedded_sequence( + x, + fps=fps, + padding_mask=padding_mask, + latent_condition=latent_condition, + latent_condition_sigma=latent_condition_sigma, + trajectory=trajectory, + frame_repeat=frame_repeat, + ) + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if scalar_feature is not None: + raise NotImplementedError("Scalar feature is not implemented yet.") + timesteps_B_D = timesteps_B_D + scalar_feature.mean(dim=1) + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + # for logging purpose + self.affline_scale_log_info = affline_scale_log_info + self.affline_emb = affline_emb_B_D + self.crossattn_emb = crossattn_emb + self.crossattn_mask = crossattn_mask + + if self.use_cross_attn_mask: + crossattn_mask = crossattn_mask[:, None, None, :].to(dtype=torch.bool) # [B, 1, 1, length] + else: + crossattn_mask = None + + if self.blocks["block0"].x_format == "THWBD": + x = rearrange(x_B_T_H_W_D, "B T H W D -> T H W B D") + if extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D is not None: + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = rearrange( + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, "B T H W D -> T H W B D" + ) + crossattn_emb = rearrange(crossattn_emb, "B M D -> M B D") + + if crossattn_mask: + crossattn_mask = rearrange(crossattn_mask, "B M -> M B") + + elif self.blocks["block0"].x_format == "BTHWD": + x = x_B_T_H_W_D + else: + raise ValueError(f"Unknown x_format {self.blocks[0].x_format}") + output = { + "x": x, + "affline_emb_B_D": affline_emb_B_D, + "crossattn_emb": crossattn_emb, + "crossattn_mask": crossattn_mask, + "rope_emb_L_1_1_D": rope_emb_L_1_1_D, + "adaln_lora_B_3D": adaln_lora_B_3D, + "original_shape": original_shape, + "extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D": extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, + } + return output + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + trajectory: Optional[torch.Tensor] = None, + frame_repeat: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Prepares an embedded sequence tensor by applying positional embeddings and handling padding masks. + + Args: + x_B_C_T_H_W (torch.Tensor): video + fps (Optional[torch.Tensor]): Frames per second tensor to be used for positional embedding when required. + If None, a default value (`self.base_fps`) will be used. + padding_mask (Optional[torch.Tensor]): current it is not used + + Returns: + Tuple[torch.Tensor, Optional[torch.Tensor]]: + - A tensor of shape (B, T, H, W, D) with the embedded sequence. + - An optional positional embedding tensor, returned only if the positional embedding class + (`self.pos_emb_cls`) includes 'rope'. Otherwise, None. + + Notes: + - If `self.concat_padding_mask` is True, a padding mask channel is concatenated to the input tensor. + - The method of applying positional embeddings depends on the value of `self.pos_emb_cls`. + - If 'rope' is in `self.pos_emb_cls` (case insensitive), the positional embeddings are generated using + the `self.pos_embedder` with the shape [T, H, W]. + - If "fps_aware" is in `self.pos_emb_cls`, the positional embeddings are generated using the `self.pos_embedder` + with the fps tensor. + - Otherwise, the positional embeddings are generated without considering fps. + """ + if self.concat_padding_mask: + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + + view_indices = torch.arange(self.n_cameras).to(x_B_C_T_H_W.device) # View indices [0, 1, ..., V-1] + view_embedding = self.view_embeddings(view_indices) # Shape: [V, embedding_dim] + view_embedding = rearrange(view_embedding, "V D -> D V") + view_embedding = ( + view_embedding.unsqueeze(0).unsqueeze(3).unsqueeze(4).unsqueeze(5) + ) # Shape: [1, D, V, 1, 1, 1] + + if self.add_repeat_frame_embedding: + if frame_repeat is None: + frame_repeat = ( + torch.zeros([x_B_C_T_H_W.shape[0], view_embedding.shape[1]]) + .to(view_embedding.device) + .to(view_embedding.dtype) + ) + frame_repeat_embedding = self.repeat_frame_embedding(frame_repeat.unsqueeze(-1)) + frame_repeat_embedding = rearrange(frame_repeat_embedding, "B V D -> B D V") + view_embedding = view_embedding + frame_repeat_embedding.unsqueeze(3).unsqueeze(4).unsqueeze(5) + + x_B_C_V_T_H_W = rearrange(x_B_C_T_H_W, "B C (V T) H W -> B C V T H W", V=self.n_cameras) + view_embedding = view_embedding.expand( + x_B_C_V_T_H_W.shape[0], + view_embedding.shape[1], + view_embedding.shape[2], + x_B_C_V_T_H_W.shape[3], + x_B_C_V_T_H_W.shape[4], + x_B_C_V_T_H_W.shape[5], + ) # Shape: [B, V, 3, t, H, W] + if self.concat_traj_embedding: + traj_emb = self.traj_embeddings(trajectory) + traj_emb = traj_emb.unsqueeze(2).unsqueeze(3).unsqueeze(4).unsqueeze(5) + traj_emb = traj_emb.expand( + x_B_C_V_T_H_W.shape[0], + traj_emb.shape[1], + view_embedding.shape[2], + x_B_C_V_T_H_W.shape[3], + x_B_C_V_T_H_W.shape[4], + x_B_C_V_T_H_W.shape[5], + ) # Shape: [B, V, 3, t, H, W] + + x_B_C_V_T_H_W = torch.cat([x_B_C_V_T_H_W, view_embedding, traj_emb], dim=1) + else: + x_B_C_V_T_H_W = torch.cat([x_B_C_V_T_H_W, view_embedding], dim=1) + + x_B_C_T_H_W = rearrange(x_B_C_V_T_H_W, " B C V T H W -> B C (V T) H W", V=self.n_cameras) + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None, extra_pos_emb + + +class MultiCameraVideoExtendGeneralDIT(MultiCameraGeneralDIT): + def __init__(self, *args, in_channels, **kwargs): + # extra channel for video condition mask + super().__init__(*args, in_channels=in_channels + 1, **kwargs) + log.info(f"VideoExtendGeneralDIT in_channels: {in_channels + 1}") + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + fps: Optional[torch.Tensor] = None, + image_size: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + video_cond_bool: Optional[torch.Tensor] = None, + condition_video_indicator: Optional[torch.Tensor] = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + condition_video_augment_sigma: Optional[torch.Tensor] = None, + condition_video_pose: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """Args: + condition_video_augment_sigma: (B) tensor of sigma value for the conditional input augmentation + condition_video_pose: (B, 1, T, H, W) tensor of pose condition + """ + B, C, T, H, W = x.shape + + if data_type == DataType.VIDEO: + assert condition_video_input_mask is not None, ( + "condition_video_input_mask is required for video data type; check if your model_obj is extend_model.FSDPDiffusionModel or the base DiffusionModel" + ) + input_list = [x, condition_video_input_mask] + if condition_video_pose is not None: + if condition_video_pose.shape[2] > T: + log.warning( + f"condition_video_pose has more frames than the input video: {condition_video_pose.shape} > {x.shape}" + ) + condition_video_pose = condition_video_pose[:, :, :T, :, :].contiguous() + input_list.append(condition_video_pose) + x = torch.cat( + input_list, + dim=1, + ) + + return super().forward( + x=x, + timesteps=timesteps, + crossattn_emb=crossattn_emb, + crossattn_mask=crossattn_mask, + fps=fps, + image_size=image_size, + padding_mask=padding_mask, + scalar_feature=scalar_feature, + data_type=data_type, + condition_video_augment_sigma=condition_video_augment_sigma, + **kwargs, + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit_video_conditioned.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit_video_conditioned.py new file mode 100644 index 00000000..8c991cd1 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/networks/general_dit_video_conditioned.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import torch +from cosmos1.models.diffusion.conditioner import DataType +from cosmos1.models.diffusion.module.blocks import TimestepEmbedding, Timesteps +from cosmos1.models.diffusion.networks.general_dit import GeneralDIT +from cosmos1.utils import log +from einops import rearrange +from torch import nn + + +class VideoExtendGeneralDIT(GeneralDIT): + def __init__(self, *args, in_channels=16 + 1, add_augment_sigma_embedding=False, **kwargs): + self.add_augment_sigma_embedding = add_augment_sigma_embedding + + # extra channel for video condition mask + super().__init__(*args, in_channels=in_channels, **kwargs) + log.debug(f"VideoExtendGeneralDIT in_channels: {in_channels}") + + def build_additional_timestamp_embedder(self): + super().build_additional_timestamp_embedder() + if self.add_augment_sigma_embedding: + log.info("Adding augment sigma embedding") + self.augment_sigma_embedder = nn.Sequential( + Timesteps(self.model_channels), + TimestepEmbedding(self.model_channels, self.model_channels, use_adaln_lora=self.use_adaln_lora), + ) + + def initialize_weights(self): + if self.add_augment_sigma_embedding: + # Initialize timestep embedding for augment sigma + nn.init.normal_(self.augment_sigma_embedder[1].linear_1.weight, std=0.02) + if self.augment_sigma_embedder[1].linear_1.bias is not None: + nn.init.constant_(self.augment_sigma_embedder[1].linear_1.bias, 0) + nn.init.normal_(self.augment_sigma_embedder[1].linear_2.weight, std=0.02) + if self.augment_sigma_embedder[1].linear_2.bias is not None: + nn.init.constant_(self.augment_sigma_embedder[1].linear_2.bias, 0) + + super().initialize_weights() # Call this last since it wil call TP weight init + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + fps: Optional[torch.Tensor] = None, + image_size: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + video_cond_bool: Optional[torch.Tensor] = None, + condition_video_indicator: Optional[torch.Tensor] = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + condition_video_augment_sigma: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """Forward pass of the video-conditioned DIT model. + + Args: + x: Input tensor of shape (B, C, T, H, W) + timesteps: Timestep tensor of shape (B,) + crossattn_emb: Cross attention embeddings of shape (B, N, D) + crossattn_mask: Optional cross attention mask of shape (B, N) + fps: Optional frames per second tensor + image_size: Optional image size tensor + padding_mask: Optional padding mask tensor + scalar_feature: Optional scalar features tensor + data_type: Type of data being processed (default: DataType.VIDEO) + video_cond_bool: Optional video conditioning boolean tensor + condition_video_indicator: Optional video condition indicator tensor + condition_video_input_mask: Required mask tensor for video data type + condition_video_augment_sigma: Optional sigma values for conditional input augmentation + **kwargs: Additional keyword arguments + + Returns: + torch.Tensor: Output tensor + """ + B, C, T, H, W = x.shape + + if data_type == DataType.VIDEO: + assert condition_video_input_mask is not None, "condition_video_input_mask is required for video data type" + + input_list = [x, condition_video_input_mask] + x = torch.cat( + input_list, + dim=1, + ) + + return super().forward( + x=x, + timesteps=timesteps, + crossattn_emb=crossattn_emb, + crossattn_mask=crossattn_mask, + fps=fps, + image_size=image_size, + padding_mask=padding_mask, + scalar_feature=scalar_feature, + data_type=data_type, + condition_video_augment_sigma=condition_video_augment_sigma, + **kwargs, + ) + + def forward_before_blocks( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + crossattn_mask: Optional[torch.Tensor] = None, + fps: Optional[torch.Tensor] = None, + image_size: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + scalar_feature: Optional[torch.Tensor] = None, + data_type: Optional[DataType] = DataType.VIDEO, + latent_condition: Optional[torch.Tensor] = None, + latent_condition_sigma: Optional[torch.Tensor] = None, + condition_video_augment_sigma: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + Args: + x: (B, C, T, H, W) tensor of spatial-temp inputs + timesteps: (B, ) tensor of timesteps + crossattn_emb: (B, N, D) tensor of cross-attention embeddings + crossattn_mask: (B, N) tensor of cross-attention masks + + condition_video_augment_sigma: (B, T) tensor of sigma value for the conditional input augmentation + """ + del kwargs + assert isinstance(data_type, DataType), ( + f"Expected DataType, got {type(data_type)}. We need discuss this flag later." + ) + original_shape = x.shape + x_B_T_H_W_D, rope_emb_L_1_1_D, extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = self.prepare_embedded_sequence( + x, + fps=fps, + padding_mask=padding_mask, + latent_condition=latent_condition, + latent_condition_sigma=latent_condition_sigma, + ) + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if scalar_feature is not None: + raise NotImplementedError("Scalar feature is not implemented yet.") + + if self.add_augment_sigma_embedding: + if condition_video_augment_sigma is None: + # Handling image case + # Note: for video case, when there is not condition frames, we also set it as zero, see extend_model augment_conditional_latent_frames function + assert data_type == DataType.IMAGE, "condition_video_augment_sigma is required for video data type" + condition_video_augment_sigma = torch.zeros_like(timesteps.flatten()) + + affline_augment_sigma_emb_B_D, _ = self.augment_sigma_embedder(condition_video_augment_sigma.flatten()) + affline_emb_B_D = affline_emb_B_D + affline_augment_sigma_emb_B_D + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + if self.use_cross_attn_mask: + crossattn_mask = crossattn_mask[:, None, None, :].to(dtype=torch.bool) # [B, 1, 1, length] + else: + crossattn_mask = None + + x = rearrange(x_B_T_H_W_D, "B T H W D -> T H W B D") + if extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D is not None: + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D = rearrange( + extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, "B T H W D -> T H W B D" + ) + crossattn_emb = rearrange(crossattn_emb, "B M D -> M B D") + if crossattn_mask: + crossattn_mask = rearrange(crossattn_mask, "B M -> M B") + + output = { + "x": x, + "affline_emb_B_D": affline_emb_B_D, + "crossattn_emb": crossattn_emb, + "crossattn_mask": crossattn_mask, + "rope_emb_L_1_1_D": rope_emb_L_1_1_D, + "adaln_lora_B_3D": adaln_lora_B_3D, + "original_shape": original_shape, + "extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D": extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D, + } + return output diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/inference.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/inference.py new file mode 100644 index 00000000..0b5e2c4b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/inference.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import List, Optional, TypedDict + +import torch +from cosmos1.models.autoregressive.model import AutoRegressiveModel +from cosmos1.models.autoregressive.tokenizer.image_text_tokenizer import ImageTextTokenizer +from cosmos1.models.autoregressive.tokenizer.text_tokenizer import TextTokenizer + + +class ChatPrediction(TypedDict, total=False): + tokens: List[str] # not required + logprobs: List[float] # not required + + +def chat_completion( + model: AutoRegressiveModel, + dialogs: List, + seed: int = None, + temperature: float = 0.01, + top_k: int = None, + top_p: float = None, + max_gen_len: Optional[int] = None, + num_gen_seq: int = 1, + logprobs: bool = False, + generation_prefix: str = "", + compile_sampling: bool = False, + compile_prefill: bool = False, + stop_tokens=None, + verbose: bool = False, +) -> List[ChatPrediction]: + """ + Generate assistant responses for a list of conversational dialogs using the language generation model. + + Args: + model (AutoRegressiveModel): The language generation model. + dialogs (List): List of conversational dialogs, where each dialog is a list of messages. + NOTE if you are using a VLM, all dialogs must either all have images ("image" field) or all be pure text. + temperature (float, optional): Temperature value for controlling randomness in sampling. Defaults to 0.01. + top_k (int, optional): Top-k probability threshold for nucleus sampling. Defaults to None. If not None, top-p sampling is ignored. + top_p (float, optional): Top-p probability threshold for nucleus sampling. Defaults to None. If not None, top-k sampling is ignored. + max_gen_len (Optional[int], optional): Maximum length of the generated response sequence. + If not provided, it's set to the model's maximum sequence length minus 1. + num_gen_seq (int, optional): Number of sequences to generate per prompt. Defaults to 1. + logprobs (bool, optional): Flag indicating whether to compute token log probabilities. Defaults to False. + generation_prefix (str, optional): Prefix to add before asking model to generate. Helpful to guide the generation. Defaults to "". + compile_sampling (bool, optional): Flag indicating whether to compile the generation function. Defaults to False. + compile_prefill (bool, optional): Flag indicating whether to compile the prefill function. Defaults to False. + stop_tokens (Set[int], optional): Set of tokens to stop generation. Defaults to None. If not None, it will override the model's stop tokens. + verbose (bool, optional): Flag indicating whether to print the generation throughput. Defaults to False. + Returns: + List[ChatPrediction]: List of chat predictions, each containing the assistant's generated response. + + Note: + This method generates assistant responses for the provided conversational dialogs. + It employs nucleus sampling to introduce controlled randomness in text generation. + If logprobs is True, token log probabilities are computed for each generated token. + """ + if max_gen_len is None: + max_gen_len = model.model.params.max_seq_len - 1 + images = None + if isinstance(model.tokenizer.text_tokenizer, ImageTextTokenizer): + # Vision-language model + prompt_dicts = [ + model.tokenizer.text_tokenizer.apply_chat_template( + dialog, generation_prefix=generation_prefix, add_generation_prompt=True + ) + for dialog in dialogs + ] + prompt_tokens = [prompt_dict["input_ids"] for prompt_dict in prompt_dicts] + num_images = sum(["pixel_values" in prompt_dict for prompt_dict in prompt_dicts]) + assert num_images in [0, len(dialogs)], "For VLM, all dialogs must either all have images or all be pure text." + if num_images > 0: + images = torch.cat([prompt_dict["pixel_values"] for prompt_dict in prompt_dicts], dim=0) + else: + images = None + elif isinstance(model.tokenizer.text_tokenizer, TextTokenizer): + # Text-only model + prompt_tokens = [ + model.tokenizer.text_tokenizer.apply_chat_template( + dialog, generation_prefix=generation_prefix, add_generation_prompt=True + ) + for dialog in dialogs + ] + else: + prompt_tokens = [model.formatter.encode_dialog_prompt(dialog) for dialog in dialogs] + + generation_tokens, generation_logprobs = model.generate( + prompt_tokens=prompt_tokens, + seed=seed, + max_gen_len=max_gen_len, + num_gen_seq=num_gen_seq, + temperature=temperature, + top_k=top_k, + top_p=top_p, + compile_sampling=compile_sampling, + compile_prefill=compile_prefill, + stop_tokens=stop_tokens, + verbose=verbose, + images=images, + ) + + if logprobs: + return [ + { + "generation": { + "role": "assistant", + "content": model.tokenizer.text_tokenizer.decode(t), + }, + "tokens": [model.tokenizer.text_tokenizer.decode([x]) for x in t], + "logprobs": logprobs_i, + } + for t, logprobs_i in zip(generation_tokens, generation_logprobs) + ] + return [ + { + "generation": { + "role": "assistant", + "content": model.tokenizer.text_tokenizer.decode(t), + }, + } + for t in generation_tokens + ] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/text2world_prompt_upsampler_inference.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/text2world_prompt_upsampler_inference.py new file mode 100644 index 00000000..c8970c01 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/text2world_prompt_upsampler_inference.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +This demo script is used to run inference for Cosmos-1.0-Prompt-Upsampler-12B-Text2World. +Command: + PYTHONPATH=$(pwd) python cosmos1/models/diffusion/prompt_upsampler/text2world_prompt_upsampler_inference.py + +""" + +import argparse +import os +import re + +from cosmos1.models.autoregressive.configs.base.model_config import create_text_model_config +from cosmos1.models.autoregressive.model import AutoRegressiveModel +from cosmos1.models.diffusion.prompt_upsampler.inference import chat_completion +from cosmos1.models.guardrail.common import presets as guardrail_presets +from cosmos1.utils import log + + +def create_prompt_upsampler(checkpoint_dir: str) -> AutoRegressiveModel: + model_config, tokenizer_config = create_text_model_config( + model_ckpt_path=os.path.join(checkpoint_dir, "model.pt"), + tokenizer_path=os.path.join(checkpoint_dir), + model_family="mistral", + model_size="12b", + is_instruct_model=True, + max_batch_size=1, + rope_dim="1D", + add_special_tokens=True, + max_seq_len=1024, + pytorch_rope_version="v1", + ) + log.debug(f"Text prompt upsampler model config: {model_config}") + + # Create and return a LLM instance + return AutoRegressiveModel.build( + model_config=model_config, + tokenizer_config=tokenizer_config, + ).to("cuda") + + +def run_chat_completion(model: AutoRegressiveModel, input: str, temperature: float = 0.01): + """ + text2world prompt upsampler model is finetuned for chat. + During training, the context window for the initial prompt upsampler models is 512 tokens. For inference, we set max_seq_len to 1024 to accommodate longer inputs. + Setting `max_gen_len` is optional as the finetuned models can naturally determine when to stop generating. + """ + + dialogs = [[{"role": "user", "content": f"Upsample the short caption to a long caption: {str(input)}"}]] + + results = chat_completion( + model, + dialogs, + max_gen_len=512, + temperature=temperature, + top_p=None, + top_k=None, + logprobs=False, + ) + upsampled_prompt = str(clean_text(results[0]["generation"]["content"])) + return upsampled_prompt + + +def clean_text(text: str) -> str: + """Clean the text by removing prefixes, suffixes, formatting markers, and normalizing whitespace.""" + # Replace all variations of newlines with a space + text = text.replace("\n", " ").replace("\r", " ") + + # Use a regex to find sections of the form '- **...**' + pattern = r"(- \*\*)(.*?)(\*\*)" + + def replacement(match: re.Match[str]) -> str: + content = match.group(2) # The text inside - ** and ** + words = re.findall(r"\w+", content) + if len(words) < 10: + # If fewer than 10 words, remove the entire '- **...**' portion + return "" + else: + # If 10 or more words, keep the entire section as it is + return match.group(0) + + text = re.sub(pattern, replacement, text) + + # Remove common prefixes + prefixes = ["Caption:", "#####", "####", "- ", "* ", ","] + for prefix in prefixes: + # lstrip(prefix) won't strip entire strings, but character sets. + # For more reliable prefix removal, do: + if text.startswith(prefix): + text = text[len(prefix) :].lstrip() + + # Remove extra spaces + text = " ".join(text.split()) + + # Strip any remaining leading/trailing punctuation, whitespace, and quotes + text = text.strip(' -,*:"\'"“”') + + return text + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run prompt upsampler inference") + parser.add_argument("--input", type=str, default="A dog is playing with a ball.") + parser.add_argument("--temperature", type=float, default=0.01, help="Inference temperature") + parser.add_argument( + "--checkpoint_dir", type=str, default="checkpoints", help="Base directory containing model checkpoints" + ) + parser.add_argument( + "--prompt_upsampler_dir", + type=str, + default="Cosmos-1.0-Prompt-Upsampler-12B-Text2World", + help="Prompt upsampler weights directory relative to checkpoint_dir", + ) + parser.add_argument( + "--guardrail_dir", + type=str, + default="Cosmos-1.0-Guardrail", + help="Guardrail weights directory relative to checkpoint_dir", + ) + return parser.parse_args() + + +def main(args): + guardrail_runner = guardrail_presets.create_text_guardrail_runner( + os.path.join(args.checkpoint_dir, args.guardrail_dir) + ) + is_safe = guardrail_presets.run_text_guardrail(args.input, guardrail_runner) + if not is_safe: + log.critical("Input text prompt is not safe.") + return + + prompt_upsampler = create_prompt_upsampler(os.path.join(args.checkpoint_dir, args.prompt_upsampler_dir)) + upsampled_prompt = run_chat_completion(prompt_upsampler, args.input, temperature=args.temperature) + is_safe = guardrail_presets.run_text_guardrail(upsampled_prompt, guardrail_runner) + if not is_safe: + log.critical("Upsampled text prompt is not safe.") + return + + log.info(f"Upsampled prompt: {upsampled_prompt}") + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/video2world_prompt_upsampler_inference.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/video2world_prompt_upsampler_inference.py new file mode 100644 index 00000000..5e7321c5 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/diffusion/prompt_upsampler/video2world_prompt_upsampler_inference.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +""" +This demo script is used to run inference for Pixtral-12B. +Command: + PYTHONPATH=$(pwd) python cosmos1/models/diffusion/prompt_upsampler/video2world_prompt_upsampler_inference.py + +""" + +import argparse +import os +from math import ceil + +from cosmos1.models.autoregressive.configs.base.model_config import create_vision_language_model_config +from cosmos1.models.autoregressive.model import AutoRegressiveModel +from cosmos1.models.diffusion.prompt_upsampler.inference import chat_completion +from cosmos1.models.guardrail.common import presets as guardrail_presets +from cosmos1.utils import log +from cosmos1.utils.io import load_from_fileobj +from PIL import Image + + +def create_vlm_prompt_upsampler( + checkpoint_dir: str, tokenizer_ckpt_path: str = "mistral-community/pixtral-12b" +) -> AutoRegressiveModel: + """ + Load the fine-tuned pixtral model for SimReady. + If pixtral_ckpt is not provided, use the pretrained checkpoint. + """ + model_ckpt_path = os.path.join(checkpoint_dir, "model.pt") + model_config, tokenizer_config = create_vision_language_model_config( + model_ckpt_path=model_ckpt_path, + tokenizer_ckpt_path=tokenizer_ckpt_path, + model_family="pixtral", + model_size="12b", + is_instruct_model=True, + max_batch_size=1, + max_seq_len=4300, + pytorch_rope_version="v1", + ) + # during instantiate, the weights will be downloaded (if not already cached) and loaded + return AutoRegressiveModel.build( + model_config=model_config, + tokenizer_config=tokenizer_config, + ).to("cuda") + + +def resize_image(image: Image.Image, max_size: int = 1024) -> Image.Image: + """ + Ensure that the image is no larger than max_size in both dimensions. + """ + image_width, image_height = image.size + max_width, max_height = max_size, max_size + ratio = max(image_width / max_width, image_height / max_height) + if ratio > 1: + image = image.resize((ceil(image_width / ratio), ceil(image_height / ratio))) + return image + + +def prepare_dialog(image_or_video_path: str) -> list[dict]: + if image_or_video_path.endswith(".mp4"): + video_np, _ = load_from_fileobj(image_or_video_path, format="mp4") + image_frame = video_np[-1] + image = Image.fromarray(image_frame) + else: + image: Image.Image = Image.open(image_or_video_path) + + image = resize_image(image, max_size=1024) + prompt = """\ +Your task is to transform a given prompt into a refined and concise video description, no more than 150 words. +Focus only on the content, no filler words or descriptions on the style. Never mention things outside the video. + """.strip() + + return [ + { + "role": "user", + "content": "[IMG]\n" + prompt, + "images": [image], + } + ] + + +def run_chat_completion(pixtral: AutoRegressiveModel, dialog: list[dict], **inference_args) -> str: + default_args = { + "max_gen_len": 400, + "temperature": 0, + "top_p": 0.9, + "logprobs": False, + "compile_sampling": False, + "compile_prefill": False, + } + default_args.update(inference_args) + results = chat_completion( + pixtral, + [dialog], + **default_args, + ) + assert len(results) == 1 + upsampled_prompt = str(results[0]["generation"]["content"]) + return upsampled_prompt + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run prompt upsampler inference") + parser.add_argument( + "--image_or_video_path", type=str, default="cosmos1/models/diffusion/assets/v1p0/video2world_input0.jpg" + ) + parser.add_argument("--temperature", type=float, default=0.01, help="Inference temperature") + parser.add_argument("--top_p", type=float, default=0.9, help="Top-p value for top-p sampling") + parser.add_argument( + "--checkpoint_dir", type=str, default="checkpoints", help="Base directory containing model checkpoints" + ) + parser.add_argument( + "--prompt_upsampler_dir", + type=str, + default="Pixtral-12B", + help="Prompt upsampler weights directory relative to checkpoint_dir", + ) + parser.add_argument( + "--guardrail_dir", + type=str, + default="Cosmos-1.0-Guardrail", + help="Guardrail weights directory relative to checkpoint_dir", + ) + return parser.parse_args() + + +def main(args): + guardrail_runner = guardrail_presets.create_text_guardrail_runner( + os.path.join(args.checkpoint_dir, args.guardrail_dir) + ) + + pixtral = create_vlm_prompt_upsampler(os.path.join(args.checkpoint_dir, args.prompt_upsampler_dir)) + dialog = prepare_dialog(args.image_or_video_path) + upsampled_prompt = run_chat_completion( + pixtral, + dialog, + max_gen_len=400, + temperature=args.temperature, + top_p=args.top_p, + logprobs=False, + ) + is_safe = guardrail_presets.run_text_guardrail(upsampled_prompt, guardrail_runner) + if not is_safe: + log.critical("Upsampled text prompt is not safe.") + return + + log.info(f"Upsampled prompt: {upsampled_prompt}") + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/README.md new file mode 100644 index 00000000..df170b5b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/README.md @@ -0,0 +1,21 @@ +# Cosmos Guardrail + +This page outlines a set of tools to ensure content safety in Cosmos. For implementation details, please consult the [Cosmos paper](https://research.nvidia.com/publication/2025-01_cosmos-world-foundation-model-platform-physical-ai). + +## Overview + +Our guardrail system consists of two stages: pre-Guard and post-Guard. + +Cosmos pre-Guard models are applied to text input, including input prompts and upsampled prompts. + +* Blocklist: a keyword list checker for detecting harmful keywords +* Aegis: an LLM-based approach for blocking harmful prompts + +Cosmos post-Guard models are applied to video frames generated by Cosmos models. + +* Video Content Safety Filter: a classifier trained to distinguish between safe and unsafe video frames +* Face Blur Filter: a face detection and blurring module + +## Usage + +Cosmos Guardrail models are integrated into the diffusion and autoregressive world generation pipelines in this repo. Check out the [Cosmos Diffusion Documentation](../diffusion/README.md) and [Cosmos Autoregressive Documentation](../autoregressive/README.md) to download the Cosmos Guardrail checkpoints and run the end-to-end demo scripts with our Guardrail models. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/aegis.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/aegis.py new file mode 100644 index 00000000..4f5be095 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/aegis.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse + +import torch +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +from cosmos1.models.guardrail.aegis.categories import UNSAFE_CATEGORIES +from cosmos1.models.guardrail.common.core import ContentSafetyGuardrail, GuardrailRunner +from cosmos1.utils import log, misc + + +SAFE = misc.Color.green("SAFE") +UNSAFE = misc.Color.red("UNSAFE") + +DEFAULT_CHECKPOINT_DIR = "checkpoints/Cosmos-1.0-Guardrail/aegis" + + +class Aegis(ContentSafetyGuardrail): + def __init__( + self, + checkpoint_dir: str = DEFAULT_CHECKPOINT_DIR, + device="cuda" if torch.cuda.is_available() else "cpu", + ) -> None: + self.checkpoint_dir = checkpoint_dir + self.device = device + self.dtype = torch.bfloat16 + base_model_id = "meta-llama/LlamaGuard-7b" + aegis_adapter = "nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0" + base_model = AutoModelForCausalLM.from_pretrained(base_model_id, cache_dir=self.checkpoint_dir) + self.tokenizer = AutoTokenizer.from_pretrained(base_model_id, cache_dir=self.checkpoint_dir) + self.model = PeftModel.from_pretrained(base_model, aegis_adapter, cache_dir=self.checkpoint_dir) + self.model.to(self.device, dtype=self.dtype).eval() + + def get_moderation_prompt(self, user_prompt: str) -> str: + """Create the moderation prompt for the Aegis model.""" + full_prompt = f"""[INST] Task: Check if there is unsafe content in 'User' messages in conversations according our safety policy with the below categories. + + + {"".join([f"{category['instruction']}" for category in UNSAFE_CATEGORIES])} + + + + + """ + full_prompt += f"User: {user_prompt}\n\n" + full_prompt += """ + + Provide your safety assessment for User in the above conversation: + - First line must read 'safe' or 'unsafe'. + - If unsafe, a second line must include a comma-separated list of violated categories. [/INST]""" + + return full_prompt + + def get_aegis_block_message(self, moderation_output: str) -> str: + """Extract the blocked category and reason from the Aegis model output.""" + block_msg = "Prompt blocked by Aegis." + try: + # Extract blocked category and reason + if len(moderation_output.split("\n")) >= 2: + moderation_output_list = moderation_output.split("\n") + blocked_category = int(moderation_output_list[1].lstrip("O")) - 1 + if 0 <= blocked_category < len(UNSAFE_CATEGORIES): + blocked_reason = UNSAFE_CATEGORIES[blocked_category]["blocked_reason"] + blocked_category_name = UNSAFE_CATEGORIES[blocked_category]["category"] + block_msg = f"{blocked_category_name}: {blocked_reason}" + except Exception as e: + log.warning(f"Unable to extract blocked category and reason from Aegis output: {e}") + return block_msg + + def filter_aegis_output(self, prompt: str) -> tuple[bool, str]: + """Filter the Aegis model output and return the safety status and message.""" + full_prompt = self.get_moderation_prompt(prompt) + inputs = self.tokenizer([full_prompt], add_special_tokens=False, return_tensors="pt").to(self.device) + output = self.model.generate(**inputs, max_new_tokens=100, pad_token_id=self.tokenizer.eos_token_id) + prompt_len = inputs["input_ids"].shape[-1] + moderation_output = self.tokenizer.decode(output[0][prompt_len:], skip_special_tokens=True) + + if "unsafe" in moderation_output.lower(): + block_msg = self.get_aegis_block_message(moderation_output) + return False, block_msg + else: + return True, "" + + def is_safe(self, prompt: str) -> tuple[bool, str]: + """Check if the input prompt is safe according to the Aegis model.""" + try: + return self.filter_aegis_output(prompt) + except Exception as e: + log.error(f"Unexpected error occurred when running Aegis guardrail: {e}") + return True, "Unexpected error occurred when running Aegis guardrail." + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--prompt", type=str, required=True, help="Input prompt") + parser.add_argument( + "--checkpoint_dir", + type=str, + help="Path to the Aegis checkpoint folder", + default=DEFAULT_CHECKPOINT_DIR, + ) + return parser.parse_args() + + +def main(args): + aegis = Aegis(checkpoint_dir=args.checkpoint_dir) + runner = GuardrailRunner(safety_models=[aegis]) + with misc.timer("aegis safety check"): + safety, message = runner.run_safety_check(args.prompt) + log.info(f"Input is: {'SAFE' if safety else 'UNSAFE'}") + log.info(f"Message: {message}") if not safety else None + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/categories.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/categories.py new file mode 100644 index 00000000..f9236da3 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/aegis/categories.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VIOLENCE = { + "category": "violence", + "blocked_reason": """This prompt is blocked due to the presence of violent content.""", + "instruction": """O1: Violence. +Should not +- Help people plan or engage in violence. +- Encourage people to engage in violence. +- Contain unpleasant vivid violent imagery or violent description that arouses discomfort in the reader. +Can +- Provide information on violence. +- Discuss the topics of violence, and can discuss historical events involving violence. +""", +} + +SEXUAL = { + "category": "sexual", + "blocked_reason": """This prompt is blocked due to the presence of sexual content.""", + "instruction": """O2: Sexual. +Should not +- Engage in sexually explicit (i.e., erotic) conversations and should not encourage people to engage in sexual activities. +- Engage or encourage sexual activity that is considered illegal. +Can +- Discuss topics related to sexuality and sexual education. +- Provide general information regarding sexual activities. +- Provide guidance on topics related to sexual health. +""", +} + +CRIMINAL_PLANNING = { + "category": "criminal_planning", + "blocked_reason": """This prompt is blocked due to the presence of criminal planning/confessions content.""", + "instruction": """O3: Criminal Planning/Confessions. +Should not +- Help people plan or engage in criminal activities like theft, kidnapping, or financial crimes, especially when the person indicates that they may act on the information provided to them. +- Encourage people to engage in criminal activities or condone criminal behavior. +- Have a confession of crime that was committed or will be committed in future. +Can +- Help people understand how crimes are committed, provided that the response is not so detailed that it could be used to provide a roadmap for future crimes. +- Help people understand how systems work that are used to prevent crimes, provided those descriptions do not provide detailed information on how to defeat those systems. +""", +} + +WEAPONS = { + "category": "weapons", + "blocked_reason": """This prompt is blocked due to the presence of weapons content.""", + "instruction": """O4: Guns and Illegal Weapons. +Should not +- Assist in the planning or execution of firearm or illegal weapons crimes, especially when the person indicates that they may act on the information provided to them. +- Encourage people to engage in firearm or illegal weapons crimes or condone such behavior. +- Assist or encourage people to execute or own other illegal weapons for example biological agents, or chemical weapons, including instructions on how to create or use them. +Can +- Discuss firearms and the arguments for and against firearm ownership. +""", +} + +SUBSTANCE_ABUSE = { + "category": "substance_abuse", + "blocked_reason": """This prompt is blocked due to the presence of substance abuse content.""", + "instruction": """O5: Controlled/Regulated Substances. +Should not +- Encourage or assist people in the trafficking, consumption, or use of regulated or controlled substances, including but not limited to illegal drugs, tobacco, alcohol, and cannabis. +- Assist or encourage people to create such substances when it is illegal to do so. +Can +- Discuss the use of regulated or controlled substances, and can provide general information on those substances (ex: what their chemical composition is and what their effects are). +- Provide information on how those substances are manufactured, provided that the information is not so detailed as to provide step-by-step instructions for their creation. +""", +} + +SUICIDE = { + "category": "suicide", + "blocked_reason": "This prompt is blocked due to the presence of suicide and self harm content.", + "instruction": """O6: Suicide and Self Harm. +Should not +- Encourage people to harm themselves. +- Romanticize or condone self-harm or suicide. +- Provide information regarding the methods of suicide or self-harm. +- Provide any information or advice, other than information regarding appropriate health resources, to people who express or imply that they wish to harm themselves. +- Contain confessions or intent of self harm +Can +- Provide general information regarding self-harm and suicide (ex: statistics, the mental health challenges faced by those who take these actions). +Should +- Provide information regarding appropriate health resources to people who imply or express the desire to harm themselves. +""", +} + +CSAM = { + "category": "csam", + "blocked_reason": "This prompt contains material related to Child Sexual Abuse", + "instruction": """O7: Sexual (minor). +Should not +- Assist, engage or encourage intention to engage in sexual activity with an underage/minor. +- Discuss or plan sexual acts, sharing of media of sexual nature, personal information or intention to meet in person for sexual activity with a minor. +- Should not include discussion where private information is shared with underage minors for purposes of acting on them and taking actions that are sexual in nature including meeting or sharing media. +- Should not discuss child abuse or engage, encourage, assist in sexual grooming/trafficking of children. +Can +- Provide information about sex education for use in schools. +""", +} + +HATRED = { + "category": "hatred", + "blocked_reason": "This prompt is blocked due to the presence of hate speech content.", + "instruction": """O8: Hate/Identity Hate. +Should not +- Express hateful, derogatory, or demeaning sentiments against people based on sensitive personal characteristics like their +race, color, ethnicity, religion, national origin, sexual orientation, gender, gender identity, or disability or caste. +- Encourage discrimination against people based on those characteristics and should not use slurs or create insults based on those characteristics. +- Should not discuss prejudice towards a group classified as protected according to the law of Germany, UK, USA or any other country. +- Should not contain slurs. +- Should not contain text that is distasteful, offensive, or hateful that causes others to feel grief, anger, or fear. +Can +- Provide information on discrimination. +- Discuss the topics of discrimination, and can discuss historical events involving discrimination. +- Discuss neutral statements referring to group identity and contextualized hate speech (that talks about hate but does not incite hate). +""", +} + +HARASSMENT = { + "category": "harassment", + "blocked_reason": "This prompt is blocked due to the presence of harassment content.", + "instruction": """O10: Harassment. +Should not +- Have content that may be used to torment or annoy individuals in real life, or make harassment more likely to occur. +- Engage in harassment, misconduct and unlawful discrimination +- Single out an individual or marginalization and/or retaliation based on protected characteristics like Race, Color, Gender, Sex, +Sexual orientation, Gender identity, and gender expression, National origin, Ethnicity, Disability (including being regarded as disabled), +Religion, Age (40+), Pregnancy (including pregnancy, childbirth or related medical conditions), Genetic information, Military or veteran status, Citizenship status, Political activity, or affiliation, +- Taking or requesting statutorily protected leave Body characteristics. +- Promise to give a benefit, or a threat to retaliate or take an adverse action based on a response to harassment and misconduct. +- Bullying. +- Being derogatory and humiliating toward an individual but not necessarily protected characteristics under law. +- Making rude or insulting comments, demeaning, and objectifying terms toward an individual. +Can +- Discuss negative impacts of harassment and bullying based on protected characteristics like race, color, gender, sex, nationality, within a contextualized setting. +- Provide information on protected characteristics or refers, cites material that describes what protected characteristics are. +- Provide information or refer or cite material that informs what harassment is. +""", +} + +THREAT = { + "category": "threat", + "blocked_reason": "This prompt is blocked due to the presence of threat content.", + "instruction": """O11: Threat. +Should not +- Discuss a communication of intent to inflict harm or loss on another person. +- Should not contain discussion of intention of intimidation which is a tactic used between conflicting parties to make the other timid or psychologically insecure for coercion or control. +- Should not contain intention of knowingly putting another person in fear of bodily injury. +- Should not assist or plan injury, physical or mental damage, act or instance of injury, or material or any kind of loss to a person. +- Should not contain threats that are forbidden by law as intent to obtain a monetary advantage or to compel a person to act against their will. +Can +- Provide information about threat is and the meaning of threat. +""", +} + +PROFANITY = { + "category": "profanity", + "blocked_reason": "This prompt is blocked due to the presence of profanity content.", + "instruction": """O12: Profanity. +Should not +- Contain swear words, curse words, or other obscene or profane language +- This includes offensive words used without any intention to act on them or without any explicit targeted recipient +""", +} + +UNSAFE_CATEGORIES = [ + VIOLENCE, + SEXUAL, + CRIMINAL_PLANNING, + WEAPONS, + SUBSTANCE_ABUSE, + SUICIDE, + CSAM, + HATRED, + HARASSMENT, + THREAT, + PROFANITY, +] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/blocklist.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/blocklist.py new file mode 100644 index 00000000..024b5cbd --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/blocklist.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os +import re +import string +from difflib import SequenceMatcher + +import nltk +from better_profanity import profanity + +from cosmos1.models.guardrail.blocklist.utils import read_keyword_list_from_dir, to_ascii +from cosmos1.models.guardrail.common.core import ContentSafetyGuardrail, GuardrailRunner +from cosmos1.utils import log, misc + + +DEFAULT_CHECKPOINT_DIR = "checkpoints/Cosmos-1.0-Guardrail/blocklist" +CENSOR = misc.Color.red("*") + + +class Blocklist(ContentSafetyGuardrail): + def __init__( + self, + checkpoint_dir: str = DEFAULT_CHECKPOINT_DIR, + guardrail_partial_match_min_chars: int = 4, + guardrail_partial_match_letter_count: float = 0.5, + ) -> None: + nltk.data.path.append(os.path.join(checkpoint_dir, "nltk_data")) + self.lemmatizer = nltk.WordNetLemmatizer() + self.profanity = profanity + self.checkpoint_dir = checkpoint_dir + self.guardrail_partial_match_min_chars = guardrail_partial_match_min_chars + self.guardrail_partial_match_letter_count = guardrail_partial_match_letter_count + + # Load blocklist and whitelist keywords + self.blocklist_words = read_keyword_list_from_dir(os.path.join(self.checkpoint_dir, "custom")) + self.whitelist_words = read_keyword_list_from_dir(os.path.join(self.checkpoint_dir, "whitelist")) + self.exact_match_words = read_keyword_list_from_dir(os.path.join(self.checkpoint_dir, "exact_match")) + + self.profanity.load_censor_words(custom_words=self.blocklist_words, whitelist_words=self.whitelist_words) + log.debug(f"Loaded {len(self.blocklist_words)} words/phrases from blocklist") + log.debug(f"Whitelisted {len(self.whitelist_words)} words/phrases from whitelist") + log.debug(f"Loaded {len(self.exact_match_words)} exact match words/phrases from blocklist") + + def uncensor_whitelist(self, input_prompt: str, censored_prompt: str) -> str: + """Explicitly uncensor words that are in the whitelist.""" + input_words = input_prompt.split() + censored_words = censored_prompt.split() + whitelist_words = set(self.whitelist_words) + for i, token in enumerate(input_words): + if token.strip(string.punctuation).lower() in whitelist_words: + censored_words[i] = token + censored_prompt = " ".join(censored_words) + return censored_prompt + + def censor_prompt(self, input_prompt: str) -> tuple[bool, str]: + """Censor the prompt using the blocklist with better-profanity fuzzy matching. + + Args: + input_prompt: input prompt to censor + + Returns: + bool: True if the prompt is blocked, False otherwise + str: A message indicating why the prompt was blocked + """ + censored_prompt = self.profanity.censor(input_prompt, censor_char=CENSOR) + # Uncensor whitelisted words that were censored from blocklist fuzzy matching + censored_prompt = self.uncensor_whitelist(input_prompt, censored_prompt) + if CENSOR in censored_prompt: + return True, f"Prompt blocked by censorship: Censored Prompt: {censored_prompt}" + return False, "" + + @staticmethod + def check_partial_match( + normalized_prompt: str, normalized_word: str, guardrail_partial_match_letter_count: float + ) -> tuple[bool, str]: + """ + Check robustly if normalized word and the matching target have a difference of up to guardrail_partial_match_letter_count characters. + + Args: + normalized_prompt: a string with many words + normalized_word: a string with one or multiple words, its length is smaller than normalized_prompt + guardrail_partial_match_letter_count: maximum allowed difference in characters (float to allow partial characters) + + Returns: + bool: True if a match is found, False otherwise + str: A message indicating why the prompt was blocked + """ + prompt_words = normalized_prompt.split() + word_length = len(normalized_word.split()) + max_similarity_ratio = (len(normalized_word) - float(guardrail_partial_match_letter_count)) / float( + len(normalized_word) + ) + + for i in range(len(prompt_words) - word_length + 1): + # Extract a substring from the prompt with the same number of words as the normalized_word + substring = " ".join(prompt_words[i : i + word_length]) + similarity_ratio = SequenceMatcher(None, substring, normalized_word).ratio() + if similarity_ratio >= max_similarity_ratio: + return ( + True, + f"Prompt blocked by partial match blocklist: Prompt: {normalized_prompt}, Partial Match Word: {normalized_word}", + ) + + return False, "" + + @staticmethod + def check_against_whole_word_blocklist( + prompt: str, + blocklist: list[str], + guardrail_partial_match_min_chars: int = 4, + guardrail_partial_match_letter_count: float = 0.5, + ) -> bool: + """ + Check if the prompt contains any whole words from the blocklist. + The match is case insensitive and robust to multiple spaces between words. + + Args: + prompt: input prompt to check + blocklist: list of words to check against + guardrail_partial_match_min_chars: minimum number of characters in a word to check for partial match + guardrail_partial_match_letter_count: maximum allowed difference in characters for partial match + + Returns: + bool: True if a match is found, False otherwise + str: A message indicating why the prompt was blocked + """ + # Normalize spaces and convert to lowercase + normalized_prompt = re.sub(r"\s+", " ", prompt).strip().lower() + + for word in blocklist: + # Normalize spaces and convert to lowercase for each blocklist word + normalized_word = re.sub(r"\s+", " ", word).strip().lower() + + # Use word boundaries to ensure whole word match + if re.search(r"\b" + re.escape(normalized_word) + r"\b", normalized_prompt): + return True, f"Prompt blocked by exact match blocklist: Prompt: {prompt}, Exact Match Word: {word}" + + # Check for partial match if the word is long enough + if len(normalized_word) >= guardrail_partial_match_min_chars: + match, message = Blocklist.check_partial_match( + normalized_prompt, normalized_word, guardrail_partial_match_letter_count + ) + if match: + return True, message + + return False, "" + + def is_safe(self, input_prompt: str = "") -> tuple[bool, str]: + """Check if the input prompt is safe using the blocklist.""" + # Check if the input is empty + if not input_prompt: + return False, "Input is empty" + input_prompt = to_ascii(input_prompt) + + # Check full sentence for censored words + censored, message = self.censor_prompt(input_prompt) + if censored: + return False, message + + # Check lemmatized words for censored words + tokens = nltk.word_tokenize(input_prompt) + lemmas = [self.lemmatizer.lemmatize(token) for token in tokens] + lemmatized_prompt = " ".join(lemmas) + censored, message = self.censor_prompt(lemmatized_prompt) + if censored: + return False, message + + # Check for exact match blocklist words + censored, message = self.check_against_whole_word_blocklist( + input_prompt, + self.exact_match_words, + self.guardrail_partial_match_min_chars, + self.guardrail_partial_match_letter_count, + ) + if censored: + return False, message + + # If all these checks pass, the input is safe + return True, "Input is safe" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--prompt", type=str, required=True, help="Input prompt") + parser.add_argument( + "--checkpoint_dir", + type=str, + help="Path to the Blocklist checkpoint folder", + default=DEFAULT_CHECKPOINT_DIR, + ) + return parser.parse_args() + + +def main(args): + blocklist = Blocklist(checkpoint_dir=args.checkpoint_dir) + runner = GuardrailRunner(safety_models=[blocklist]) + with misc.timer("blocklist safety check"): + safety, message = runner.run_safety_check(args.prompt) + log.info(f"Input is: {'SAFE' if safety else 'UNSAFE'}") + log.info(f"Message: {message}") if not safety else None + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/utils.py new file mode 100644 index 00000000..0c721914 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/blocklist/utils.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from cosmos1.utils import log + + +def read_keyword_list_from_dir(folder_path: str) -> list[str]: + """Read keyword list from all files in a folder.""" + output_list = [] + file_list = [] + # Get list of files in the folder + for file in os.listdir(folder_path): + if os.path.isfile(os.path.join(folder_path, file)): + file_list.append(file) + + # Process each file + for file in file_list: + file_path = os.path.join(folder_path, file) + try: + with open(file_path, "r") as f: + output_list.extend([line.strip() for line in f.readlines()]) + except Exception as e: + log.error(f"Error reading file {file}: {str(e)}") + + return output_list + + +def to_ascii(prompt: str) -> str: + """Convert prompt to ASCII.""" + return re.sub(r"[^\x00-\x7F]+", " ", prompt) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/core.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/core.py new file mode 100644 index 00000000..b6c205e8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/core.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, Tuple + +import numpy as np + +from cosmos1.utils import log + + +class ContentSafetyGuardrail: + def is_safe(self, **kwargs) -> Tuple[bool, str]: + raise NotImplementedError("Child classes must implement the is_safe method") + + +class PostprocessingGuardrail: + def postprocess(self, frames: np.ndarray) -> np.ndarray: + raise NotImplementedError("Child classes must implement the postprocess method") + + +class GuardrailRunner: + def __init__( + self, + safety_models: list[ContentSafetyGuardrail] | None = None, + generic_block_msg: str = "", + generic_safe_msg: str = "", + postprocessors: list[PostprocessingGuardrail] | None = None, + ): + self.safety_models = safety_models + self.generic_block_msg = generic_block_msg + self.generic_safe_msg = generic_safe_msg if generic_safe_msg else "Prompt is safe" + self.postprocessors = postprocessors + + def run_safety_check(self, input: Any) -> Tuple[bool, str]: + """Run the safety check on the input.""" + if not self.safety_models: + log.warning("No safety models found, returning safe") + return True, self.generic_safe_msg + + for guardrail in self.safety_models: + guardrail_name = str(guardrail.__class__.__name__).upper() + log.debug(f"Running guardrail: {guardrail_name}") + safe, message = guardrail.is_safe(input) + if not safe: + reasoning = self.generic_block_msg if self.generic_block_msg else f"{guardrail_name}: {message}" + return False, reasoning + return True, self.generic_safe_msg + + def postprocess(self, frames: np.ndarray) -> np.ndarray: + """Run the postprocessing on the video frames.""" + if not self.postprocessors: + log.warning("No postprocessors found, returning original frames") + return frames + + for guardrail in self.postprocessors: + guardrail_name = str(guardrail.__class__.__name__).upper() + log.debug(f"Running guardrail: {guardrail_name}") + frames = guardrail.postprocess(frames) + return frames diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/io_utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/io_utils.py new file mode 100644 index 00000000..c47d5855 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/io_utils.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import glob +from dataclasses import dataclass + +import imageio +import numpy as np + +from cosmos1.utils import log + + +@dataclass +class VideoData: + frames: np.ndarray # Shape: [B, H, W, C] + fps: int + duration: int # in seconds + + +def get_video_filepaths(input_dir: str) -> list[str]: + """Get a list of filepaths for all videos in the input directory.""" + paths = glob.glob(f"{input_dir}/**/*.mp4", recursive=True) + paths += glob.glob(f"{input_dir}/**/*.avi", recursive=True) + paths += glob.glob(f"{input_dir}/**/*.mov", recursive=True) + paths = sorted(paths) + log.debug(f"Found {len(paths)} videos") + return paths + + +def read_video(filepath: str) -> VideoData: + """Read a video file and extract its frames and metadata.""" + try: + reader = imageio.get_reader(filepath, "ffmpeg") + except Exception as e: + raise ValueError(f"Failed to read video file: {filepath}") from e + + # Extract metadata from the video file + try: + metadata = reader.get_meta_data() + fps = metadata.get("fps") + duration = metadata.get("duration") + except Exception as e: + reader.close() + raise ValueError(f"Failed to extract metadata from video file: {filepath}") from e + + # Extract frames from the video file + try: + frames = np.array([frame for frame in reader]) + except Exception as e: + raise ValueError(f"Failed to extract frames from video file: {filepath}") from e + finally: + reader.close() + + return VideoData(frames=frames, fps=fps, duration=duration) + + +def save_video(filepath: str, frames: np.ndarray, fps: int) -> None: + """Save a video file from a sequence of frames.""" + try: + writer = imageio.get_writer(filepath, fps=fps, macro_block_size=1) + for frame in frames: + writer.append_data(frame) + except Exception as e: + raise ValueError(f"Failed to save video file to {filepath}") from e + finally: + writer.close() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/presets.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/presets.py new file mode 100644 index 00000000..8b28c554 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/common/presets.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +from cosmos1.models.guardrail.aegis.aegis import Aegis +from cosmos1.models.guardrail.blocklist.blocklist import Blocklist +from cosmos1.models.guardrail.common.core import GuardrailRunner +from cosmos1.models.guardrail.face_blur_filter.face_blur_filter import RetinaFaceFilter +from cosmos1.models.guardrail.video_content_safety_filter.video_content_safety_filter import VideoContentSafetyFilter +from cosmos1.utils import log + + +def create_text_guardrail_runner(checkpoint_dir: str) -> GuardrailRunner: + """Create the text guardrail runner.""" + blocklist_checkpoint_dir = os.path.join(checkpoint_dir, "blocklist") + aegis_checkpoint_dir = os.path.join(checkpoint_dir, "aegis") + return GuardrailRunner(safety_models=[Blocklist(blocklist_checkpoint_dir), Aegis(aegis_checkpoint_dir)]) + + +def create_video_guardrail_runner(checkpoint_dir: str) -> GuardrailRunner: + """Create the video guardrail runner.""" + video_filter_checkpoint_dir = os.path.join(checkpoint_dir, "video_content_safety_filter") + retinaface_checkpoint_path = os.path.join(checkpoint_dir, "face_blur_filter/Resnet50_Final.pth") + return GuardrailRunner( + safety_models=[VideoContentSafetyFilter(video_filter_checkpoint_dir)], + postprocessors=[RetinaFaceFilter(retinaface_checkpoint_path)], + ) + + +def run_text_guardrail(prompt: str, guardrail_runner: GuardrailRunner) -> bool: + """Run the text guardrail on the prompt, checking for content safety. + + Args: + prompt: The text prompt. + guardrail_runner: The text guardrail runner. + + Returns: + bool: Whether the prompt is safe. + """ + is_safe, message = guardrail_runner.run_safety_check(prompt) + if not is_safe: + log.critical(f"GUARDRAIL BLOCKED: {message}") + return is_safe + + +def run_video_guardrail(frames: np.ndarray, guardrail_runner: GuardrailRunner) -> np.ndarray | None: + """Run the video guardrail on the frames, checking for content safety and applying face blur. + + Args: + frames: The frames of the generated video. + guardrail_runner: The video guardrail runner. + + Returns: + The processed frames if safe, otherwise None. + """ + is_safe, message = guardrail_runner.run_safety_check(frames) + if not is_safe: + log.critical(f"GUARDRAIL BLOCKED: {message}") + return None + + frames = guardrail_runner.postprocess(frames) + return frames diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/blur_utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/blur_utils.py new file mode 100644 index 00000000..d52f69d2 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/blur_utils.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cv2 +import numpy as np + + +def pixelate_face(face_img: np.ndarray, blocks: int = 5) -> np.ndarray: + """ + Pixelate a face region by reducing resolution and then upscaling. + + Args: + face_img: Face region to pixelate + blocks: Number of blocks to divide the face into (in each dimension) + + Returns: + Pixelated face region + """ + h, w = face_img.shape[:2] + # Shrink the image and scale back up to create pixelation effect + temp = cv2.resize(face_img, (blocks, blocks), interpolation=cv2.INTER_LINEAR) + pixelated = cv2.resize(temp, (w, h), interpolation=cv2.INTER_NEAREST) + return pixelated diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/face_blur_filter.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/face_blur_filter.py new file mode 100644 index 00000000..bc0d8fd1 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/face_blur_filter.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import os + +import numpy as np +import torch +from pytorch_retinaface.data import cfg_re50 +from pytorch_retinaface.layers.functions.prior_box import PriorBox +from pytorch_retinaface.models.retinaface import RetinaFace +from torch.utils.data import DataLoader, TensorDataset +from tqdm import tqdm + +from cosmos1.models.guardrail.common.core import GuardrailRunner, PostprocessingGuardrail +from cosmos1.models.guardrail.common.io_utils import get_video_filepaths, read_video, save_video +from cosmos1.models.guardrail.face_blur_filter.blur_utils import pixelate_face +from cosmos1.models.guardrail.face_blur_filter.retinaface_utils import decode_batch, filter_detected_boxes, load_model +from cosmos1.utils import log, misc + + +DEFAULT_RETINAFACE_CHECKPOINT = "checkpoints/Cosmos-1.0-Guardrail/face_blur_filter/Resnet50_Final.pth" + +# RetinaFace model constants from https://github.com/biubug6/Pytorch_Retinaface/blob/master/detect.py +TOP_K = 5_000 +KEEP_TOP_K = 750 +NMS_THRESHOLD = 0.4 + + +class RetinaFaceFilter(PostprocessingGuardrail): + def __init__( + self, + checkpoint: str = DEFAULT_RETINAFACE_CHECKPOINT, + batch_size: int = 1, + confidence_threshold: float = 0.7, + device="cuda" if torch.cuda.is_available() else "cpu", + ) -> None: + """ + Initialize the RetinaFace model for face detection and blurring. + + Args: + checkpoint: Path to the RetinaFace checkpoint file + batch_size: Batch size for RetinaFace inference and processing + confidence_threshold: Minimum confidence score to consider a face detection + """ + self.cfg = cfg_re50 + self.batch_size = batch_size + self.confidence_threshold = confidence_threshold + self.device = device + self.dtype = torch.float32 + + # Disable loading ResNet pretrained weights + self.cfg["pretrain"] = False + self.net = RetinaFace(cfg=self.cfg, phase="test") + cpu = self.device == "cpu" + + # Load from RetinaFace pretrained checkpoint + self.net = load_model(self.net, checkpoint, cpu) + self.net.to(self.device, dtype=self.dtype).eval() + + def preprocess_frames(self, frames: np.ndarray) -> torch.Tensor: + """Preprocess a sequence of frames for face detection. + + Args: + frames: Input frames + + Returns: + Preprocessed frames tensor + """ + with torch.no_grad(): + frames_tensor = torch.from_numpy(frames).to(self.device, dtype=self.dtype) # Shape: [T, H, W, C] + frames_tensor = frames_tensor.permute(0, 3, 1, 2) # Shape: [T, C, H, W] + frames_tensor = frames_tensor[:, [2, 1, 0], :, :] # RGB to BGR to match RetinaFace model input + means = torch.tensor([104.0, 117.0, 123.0], device=self.device, dtype=self.dtype).view(1, 3, 1, 1) + frames_tensor = frames_tensor - means # Subtract mean BGR values for each channel + return frames_tensor + + def blur_detected_faces( + self, + frames: np.ndarray, + batch_loc: torch.Tensor, + batch_conf: torch.Tensor, + prior_data: torch.Tensor, + scale: torch.Tensor, + min_size: tuple[int] = (20, 20), + ) -> list[np.ndarray]: + """Blur detected faces in a batch of frames using RetinaFace predictions. + + Args: + frames: Input frames + batch_loc: Batched location predictions + batch_conf: Batched confidence scores + prior_data: Prior boxes for the video + scale: Scale factor for resizing detections + min_size: Minimum size of a detected face region in pixels + + Returns: + Processed frames with pixelated faces + """ + with torch.no_grad(): + batch_boxes = decode_batch(batch_loc, prior_data, self.cfg["variance"]) + batch_boxes = batch_boxes * scale + + blurred_frames = [] + for i, boxes in enumerate(batch_boxes): + boxes = boxes.detach().cpu().numpy() + scores = batch_conf[i, :, 1].detach().cpu().numpy() + + filtered_boxes = filter_detected_boxes( + boxes, + scores, + confidence_threshold=self.confidence_threshold, + nms_threshold=NMS_THRESHOLD, + top_k=TOP_K, + keep_top_k=KEEP_TOP_K, + ) + + frame = frames[i] + for box in filtered_boxes: + x1, y1, x2, y2 = map(int, box) + # Ignore bounding boxes smaller than the minimum size + if x2 - x1 < min_size[0] or y2 - y1 < min_size[1]: + continue + max_h, max_w = frame.shape[:2] + face_roi = frame[max(y1, 0) : min(y2, max_h), max(x1, 0) : min(x2, max_w)] + blurred_face = pixelate_face(face_roi) + frame[max(y1, 0) : min(y2, max_h), max(x1, 0) : min(x2, max_w)] = blurred_face + blurred_frames.append(frame) + + return blurred_frames + + def postprocess(self, frames: np.ndarray) -> np.ndarray: + """Blur faces in a sequence of frames. + + Args: + frames: Input frames + + Returns: + Processed frames with pixelated faces + """ + # Create dataset and dataloader + frames_tensor = self.preprocess_frames(frames) + dataset = TensorDataset(frames_tensor) + dataloader = DataLoader(dataset, batch_size=self.batch_size, shuffle=False) + processed_frames, processed_batches = [], [] + + prior_data, scale = None, None + for i, batch in enumerate(dataloader): + batch = batch[0] + h, w = batch.shape[-2:] # Batch shape: [C, H, W] + + with torch.no_grad(): + # Generate priors for the video + if prior_data is None: + priorbox = PriorBox(self.cfg, image_size=(h, w)) + priors = priorbox.forward() + priors = priors.to(self.device, dtype=self.dtype) + prior_data = priors.data + + # Get scale for resizing detections + if scale is None: + scale = torch.Tensor([w, h, w, h]) + scale = scale.to(self.device, dtype=self.dtype) + + batch_loc, batch_conf, _ = self.net(batch) + + # Blur detected faces in each batch of frames + start_idx = i * self.batch_size + end_idx = min(start_idx + self.batch_size, len(frames)) + processed_batches.append( + self.blur_detected_faces(frames[start_idx:end_idx], batch_loc, batch_conf, prior_data, scale) + ) + + processed_frames = [frame for batch in processed_batches for frame in batch] + return np.array(processed_frames) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", type=str, required=True, help="Path containing input videos") + parser.add_argument("--output_dir", type=str, required=True, help="Path for saving processed videos") + parser.add_argument( + "--checkpoint", + type=str, + help="Path to the RetinaFace checkpoint file", + default=DEFAULT_RETINAFACE_CHECKPOINT, + ) + return parser.parse_args() + + +def main(args): + filepaths = get_video_filepaths(args.input_dir) + if not filepaths: + log.error(f"No video files found in directory: {args.input_dir}") + return + + face_blur = RetinaFaceFilter(checkpoint=args.checkpoint) + postprocessing_runner = GuardrailRunner(postprocessors=[face_blur]) + os.makedirs(args.output_dir, exist_ok=True) + + for filepath in tqdm(filepaths): + video_data = read_video(filepath) + with misc.timer("face blur filter"): + frames = postprocessing_runner.postprocess(video_data.frames) + + output_path = os.path.join(args.output_dir, os.path.basename(filepath)) + save_video(output_path, frames, video_data.fps) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/retinaface_utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/retinaface_utils.py new file mode 100644 index 00000000..9dde42a3 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/face_blur_filter/retinaface_utils.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import numpy as np +import torch +from pytorch_retinaface.utils.nms.py_cpu_nms import py_cpu_nms + +from cosmos1.utils import log + + +# Adapted from https://github.com/biubug6/Pytorch_Retinaface/blob/master/detect.py +def filter_detected_boxes(boxes, scores, confidence_threshold, nms_threshold, top_k, keep_top_k): + """Filter boxes based on confidence score and remove overlapping boxes using NMS.""" + # Keep detections with confidence above threshold + inds = np.where(scores > confidence_threshold)[0] + boxes = boxes[inds] + scores = scores[inds] + + # Sort by confidence and keep top K detections + order = scores.argsort()[::-1][:top_k] + boxes = boxes[order] + scores = scores[order] + + # Run non-maximum-suppression (NMS) to remove overlapping boxes + dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False) + keep = py_cpu_nms(dets, nms_threshold) + dets = dets[keep, :] + dets = dets[:keep_top_k, :] + boxes = dets[:, :-1] + return boxes + + +# Adapted from https://github.com/biubug6/Pytorch_Retinaface/blob/master/utils/box_utils.py to handle batched inputs +def decode_batch(loc, priors, variances): + """Decode batched locations from predictions using priors and variances. + + Args: + loc (tensor): Batched location predictions for loc layers. + Shape: [batch_size, num_priors, 4] + priors (tensor): Prior boxes in center-offset form. + Shape: [num_priors, 4] + variances: (list[float]): Variances of prior boxes. + + Return: + Decoded batched bounding box predictions + Shape: [batch_size, num_priors, 4] + """ + batch_size = loc.size(0) + priors = priors.unsqueeze(0).expand(batch_size, -1, -1) + + boxes = torch.cat( + ( + priors[:, :, :2] + loc[:, :, :2] * variances[0] * priors[:, :, 2:], + priors[:, :, 2:] * torch.exp(loc[:, :, 2:] * variances[1]), + ), + dim=2, + ) + + boxes[:, :, :2] -= boxes[:, :, 2:] / 2 + boxes[:, :, 2:] += boxes[:, :, :2] + return boxes + + +# Adapted from https://github.com/biubug6/Pytorch_Retinaface/blob/master/detect.py +def _check_keys(model, pretrained_state_dict): + ckpt_keys = set(pretrained_state_dict.keys()) + model_keys = set(model.state_dict().keys()) + used_pretrained_keys = model_keys & ckpt_keys + unused_pretrained_keys = ckpt_keys - model_keys + missing_keys = model_keys - ckpt_keys + log.debug("Missing keys:{}".format(len(missing_keys))) + log.debug("Unused checkpoint keys:{}".format(len(unused_pretrained_keys))) + log.debug("Used keys:{}".format(len(used_pretrained_keys))) + assert len(used_pretrained_keys) > 0, "load NONE from pretrained checkpoint" + return True + + +# Adapted from https://github.com/biubug6/Pytorch_Retinaface/blob/master/detect.py +def _remove_prefix(state_dict, prefix): + """Old version of the model is stored with all names of parameters sharing common prefix 'module.'""" + log.debug("Removing prefix '{}'".format(prefix)) + + def f(x): + return x.split(prefix, 1)[-1] if x.startswith(prefix) else x + + return {f(key): value for key, value in state_dict.items()} + + +# Adapted from https://github.com/biubug6/Pytorch_Retinaface/blob/master/detect.py +def load_model(model, pretrained_path, load_to_cpu): + log.debug("Loading pretrained model from {}".format(pretrained_path)) + if load_to_cpu: + pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage, weights_only=True) + else: + device = torch.cuda.current_device() + pretrained_dict = torch.load( + pretrained_path, map_location=lambda storage, loc: storage.cuda(device), weights_only=True + ) + if "state_dict" in pretrained_dict.keys(): + pretrained_dict = _remove_prefix(pretrained_dict["state_dict"], "module.") + else: + pretrained_dict = _remove_prefix(pretrained_dict, "module.") + _check_keys(model, pretrained_dict) + model.load_state_dict(pretrained_dict, strict=False) + return model diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/model.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/model.py new file mode 100644 index 00000000..0c83e6c8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/model.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import attrs +import torch +import torch.nn as nn + +from cosmos1.utils.config import make_freezable + + +@make_freezable +@attrs.define(slots=False) +class ModelConfig: + input_size: int = 1152 + num_classes: int = 7 + + +class SafetyClassifier(nn.Module): + def __init__(self, input_size: int = 1024, num_classes: int = 2): + super().__init__() + self.input_size = input_size + self.num_classes = num_classes + self.layers = nn.Sequential( + nn.Linear(self.input_size, 512), + nn.BatchNorm1d(512), + nn.ReLU(), + nn.Linear(512, 256), + nn.BatchNorm1d(256), + nn.ReLU(), + nn.Linear(256, self.num_classes), + # Note: No activation function here; CrossEntropyLoss expects raw logits + ) + + def forward(self, x): + return self.layers(x) + + +class VideoSafetyModel(nn.Module): + def __init__(self, config: ModelConfig) -> None: + super().__init__() + self.config = config + self.num_classes = config.num_classes + self.network = SafetyClassifier(input_size=config.input_size, num_classes=self.num_classes) + + @torch.inference_mode() + def forward(self, data_batch: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + logits = self.network(data_batch["data"].cuda()) + return {"logits": logits} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/video_content_safety_filter.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/video_content_safety_filter.py new file mode 100644 index 00000000..73100222 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/video_content_safety_filter.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import argparse +import json +import os +from typing import Iterable, Tuple, Union + +import torch +from PIL import Image + +from cosmos1.models.guardrail.common.core import ContentSafetyGuardrail, GuardrailRunner +from cosmos1.models.guardrail.common.io_utils import get_video_filepaths, read_video +from cosmos1.models.guardrail.video_content_safety_filter.model import ModelConfig, VideoSafetyModel +from cosmos1.models.guardrail.video_content_safety_filter.vision_encoder import SigLIPEncoder +from cosmos1.utils import log, misc + + +DEFAULT_CHECKPOINT_DIR = "checkpoints/Cosmos-1.0-Guardrail/video_content_safety_filter" + + +# Define the class index to class name mapping for multi-class classification +CLASS_IDX_TO_NAME = { + 0: "Safe", + 1: "Sexual_Content", + 2: "Violence", + 3: "Drugs", + 4: "Child_Abuse", + 5: "Hate_and_Harassment", + 6: "Self-Harm", +} + + +class VideoContentSafetyFilter(ContentSafetyGuardrail): + def __init__( + self, + checkpoint_dir: str = DEFAULT_CHECKPOINT_DIR, + device="cuda" if torch.cuda.is_available() else "cpu", + ) -> None: + self.device = device + self.dtype = torch.float32 + + # Initialize the SigLIP encoder + self.encoder = SigLIPEncoder(checkpoint_dir=checkpoint_dir, device=device, dtype=self.dtype) + + # Use ModelConfig directly for inference configuration + model_config = ModelConfig(input_size=1152, num_classes=7) + + # Load the multi-class classifier + self.model = VideoSafetyModel(model_config) + safety_filter_local_path = os.path.join(checkpoint_dir, "safety_filter.pt") + checkpoint = torch.load(safety_filter_local_path, map_location=torch.device("cpu"), weights_only=True) + self.model.load_state_dict(checkpoint["model"]) + self.model.to(self.device, dtype=self.dtype).eval() + + @torch.inference_mode() + def __infer(self, pil_image: Image.Image) -> int: + """Infer the class of the image.""" + image_embs = self.encoder.encode_image(pil_image) + logits = self.model.network(image_embs) + probabilities = torch.nn.functional.softmax(logits, dim=-1) + predicted_class = torch.argmax(probabilities, dim=-1).item() + return predicted_class + + def is_safe_file(self, filepath: str) -> bool: + """Check if the video file is safe.""" + video_data = read_video(filepath) + + # Sample frames at 2 FPS + sample_rate = 2 # frames per second + frame_interval = int(video_data.fps / sample_rate) + frame_numbers = list(range(0, int(video_data.fps * video_data.duration), frame_interval)) + + is_safe = True + frame_scores = [] + + for frame_number in frame_numbers: + try: + frame = video_data.frames[frame_number] + pil_image = Image.fromarray(frame) + predicted_class = self.__infer(pil_image) + class_name = CLASS_IDX_TO_NAME.get(predicted_class, "Unknown") + frame_scores.append({"frame_number": frame_number, "class": class_name}) + + # If any frame is not "Safe", mark the video as unsafe + if predicted_class != 0: + is_safe = False + break + + except Exception as e: + log.warning(f"Warning: Failed to run safety classifier on frame_number {frame_number}. Exception: {e}") + continue + + # Prepare data for JSON + video_data = { + "filepath": filepath, + "is_safe": is_safe, + "video_length": video_data.duration, + "fps": video_data.fps, + "frame_scores": frame_scores, + } + + log.info(f"Video {filepath} is {'SAFE' if is_safe else 'UNSAFE'}.") + log.debug(f"Video data: {json.dumps(video_data, indent=4)}") + return is_safe + + def is_safe_frames(self, frames: Iterable) -> bool: + """Check if the video frames are safe.""" + is_safe = True + frame_scores = [] + + for frame_number, frame in enumerate(frames): + try: + pil_image = Image.fromarray(frame) + predicted_class = self.__infer(pil_image) + class_name = CLASS_IDX_TO_NAME.get(predicted_class, "Unknown") + frame_scores.append({"frame_number": frame_number, "class": class_name}) + + # If any frame is not "Safe", mark as not safe + if predicted_class != 0: + is_safe = False + break + + except Exception as e: + log.warning(f"Warning: Failed to run safety classifier on frame_number {frame_number}. Exception: {e}") + continue + + video_data = { + "is_safe": is_safe, + "frame_scores": frame_scores, + } + + log.debug(f"Frames data: {json.dumps(video_data, indent=4)}") + return is_safe + + def is_safe(self, input: Union[str, Iterable]) -> Tuple[bool, str]: + if isinstance(input, str): + is_safe = self.is_safe_file(input) + return is_safe, "safe video detected" if is_safe else "unsafe video detected" + elif isinstance(input, Iterable): + is_safe = self.is_safe_frames(input) + return is_safe, "safe frames detected" if is_safe else "unsafe frames detected" + else: + raise ValueError(f"Input type {type(input)} not supported.") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", type=str, required=True, help="Path containing input videos") + parser.add_argument( + "--checkpoint_dir", + type=str, + help="Path to the Video Content Safety Filter checkpoint folder", + default=DEFAULT_CHECKPOINT_DIR, + ) + return parser.parse_args() + + +def main(args): + filepaths = get_video_filepaths(args.input_dir) + if not filepaths: + log.error(f"No video files found in directory: {args.input_dir}") + return + + video_filter = VideoContentSafetyFilter(checkpoint_dir=args.checkpoint_dir) + runner = GuardrailRunner(safety_models=[video_filter], generic_safe_msg="Video is safe") + + for filepath in filepaths: + with misc.timer("video content safety filter"): + _ = runner.run_safety_check(filepath) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/vision_encoder.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/vision_encoder.py new file mode 100644 index 00000000..87074ced --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/guardrail/video_content_safety_filter/vision_encoder.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +from PIL import Image +from transformers import SiglipModel, SiglipProcessor + + +DEFAULT_CHECKPOINT_DIR = "checkpoints/Cosmos-1.0-Guardrail/video_content_safety_filter" + + +class SigLIPEncoder(torch.nn.Module): + def __init__( + self, + model_name: str = "google/siglip-so400m-patch14-384", + checkpoint_dir: str = DEFAULT_CHECKPOINT_DIR, + device="cuda" if torch.cuda.is_available() else "cpu", + dtype=torch.float32, + ) -> None: + super().__init__() + self.checkpoint_dir = checkpoint_dir + self.device = device + self.dtype = dtype + self.model = SiglipModel.from_pretrained(model_name, cache_dir=self.checkpoint_dir) + self.processor = SiglipProcessor.from_pretrained(model_name, cache_dir=self.checkpoint_dir) + self.model.to(self.device, dtype=self.dtype).eval() + + @torch.inference_mode() + def encode_image(self, input_img: Image.Image) -> torch.Tensor: + """Encode an image into a feature vector.""" + with torch.no_grad(): + inputs = self.processor(images=input_img, return_tensors="pt").to(self.device, dtype=self.dtype) + image_features = self.model.get_image_features(**inputs) + image_features /= image_features.norm(dim=-1, keepdim=True) + return image_features diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/README.md new file mode 100644 index 00000000..29b87c9e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/README.md @@ -0,0 +1,271 @@ + +# Cosmos Tokenizer: A suite of image and video neural tokenizers. + +### [Website](https://research.nvidia.com/labs/dir/cosmos-tokenizer) | [Paper](https://arxiv.org/abs/2501.03575) | [NVIDIA Cosmos](https://www.nvidia.com/en-us/ai/cosmos/) | [NVIDIA Blog](https://developer.nvidia.com/blog/state-of-the-art-multimodal-generative-ai-model-development-with-nvidia-nemo/) | [Hugging Face](https://huggingface.co/collections/nvidia/cosmos-tokenizer-672b93023add81b66a8ff8e6) | [YouTube](https://youtu.be/Soy_myOfWIU) | [TokenBench](https://github.com/NVlabs/TokenBench) + +We present [**NVIDIA Cosmos Tokenizer**](https://github.com/NVIDIA/Cosmos/blob/main/cosmos1/models/tokenizer), a suite of image and video tokenizers that advances the state-of-the-art in visual tokenization, paving the way for scalable, robust and efficient development of large auto-regressive transformers (such as LLMs) or diffusion generators. Cosmos Tokenizer is the core component of the [**NVIDIA Cosmos**](https://github.com/NVIDIA/Cosmos), a developer-first video foundation model platform designed to help Physical AI developers build their Physical AI systems better and faster. Please check out our [demo video](https://youtu.be/Soy_myOfWIU). + + +| | Continuous ( C ) | Discrete ( D ) | +| ------------------|---------------------|---------------------| +| **Images ( I )** | Cosmos-Tokenizer-CI | Cosmos-Tokenizer-DI | +| **Videos ( V )** | Cosmos-Tokenizer-CV | Cosmos-Tokenizer-DV | + + +Given an image or video, Cosmos Tokenizer outputs either continuous latents or discrete tokens. Cosmos Tokenizer achieves spatial compression rates of 8x or 16x and temporal compression factors of 4x or 8x, resulting in a total compression factor of up to 2048x (=8x16x16). +Cosmos Tokenizer delivers 8x more total compression than state-of-the-art (SOTA) methods, while simultaneously maintaining higher image quality and running up to 12x faster than the best available SOTA tokenizers. + +![Arch](assets/arch_diagram.jpg) + +## Web Demo + +* Image Tokenization [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/nvidia/Cosmos/blob/main/cosmos1/models/tokenizer/notebook/Image_Tokenization.ipynb) +* Video Tokenization [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/nvidia/Cosmos/blob/main/cosmos1/models/tokenizer/notebook/Video_Tokenization.ipynb) + +## Licenses +- **Models**: The models are licensed under [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf). Under the NVIDIA Open Model License, NVIDIA confirms: + - Models are commercially usable. + - You are free to create and distribute Derivative Models. + - NVIDIA does not claim ownership to any outputs generated using the Models or Derivative Models. +- **GitHub Code**: This repository is licensed under the [Apache 2.0 + license](https://github.com/NVIDIA/Cosmos/blob/main/LICENSE). + +## Installation + +Follow our [Installation Guide](../../../INSTALL.md) to set up the Docker environment. All commands on this page should be run inside Docker. + +## Download Pre-trained Checkpoints from Hugging Face + + +We host 12 Cosmos-Tokenizer models on [Hugging Face](https://huggingface.co/collections/nvidia/cosmos-tokenizer-672b93023add81b66a8ff8e6), with the following model names. You can use this snippet to download: +```python +from huggingface_hub import login, snapshot_download +import os + +login(token="", add_to_git_credential=True) +model_names = [ + "Cosmos-0.1-Tokenizer-CI8x8", + "Cosmos-0.1-Tokenizer-CI16x16", + "Cosmos-0.1-Tokenizer-CV4x8x8", + "Cosmos-0.1-Tokenizer-CV8x8x8", + "Cosmos-0.1-Tokenizer-CV8x16x16", + "Cosmos-0.1-Tokenizer-DI8x8", + "Cosmos-0.1-Tokenizer-DI16x16", + "Cosmos-0.1-Tokenizer-DV4x8x8", + "Cosmos-0.1-Tokenizer-DV8x8x8", + "Cosmos-0.1-Tokenizer-DV8x16x16", + "Cosmos-1.0-Tokenizer-CV8x8x8", + "Cosmos-1.0-Tokenizer-DV8x16x16", + "Cosmos-1.0-Tokenizer-CI8x8-LowRes", + "Cosmos-1.0-Tokenizer-CI16x16-LowRes", + "Cosmos-1.0-Tokenizer-DI8x8-LowRes", + "Cosmos-1.0-Tokenizer-DI16x16-LowRes", + "Cosmos-1.0-Tokenizer-CV4x8x8-LowRes", + "Cosmos-1.0-Tokenizer-DV4x8x8-LowRes", +] +for model_name in model_names: + hf_repo = "nvidia/" + model_name + local_dir = "checkpoints/" + model_name + print(f"downloading {model_name}...") + snapshot_download(repo_id=hf_repo, local_dir=local_dir) +``` +Under the checkpoint repository `checkpoints/{model_name}`, we provide the encoder, decoder and the full autoencoder JIT models. +```bash +├── Cosmos-1.0-Tokenizer-CV8x8x8/ +│ ├── encoder.jit +│ ├── decoder.jit +│ ├── autoencoder.jit +``` +## Running the codes +You can use the following example commands to encode and decode images or videos.
+For each, the same command works for both continuous and discrete tokenization. Simply provide the proper JIT-compiled ckpt to `checkpoint_enc`, `checkpoint_dec`, or the full autoencoder ckpt to `checkpoint`. + +### Encoding into Continuous Latent Space + +```python +import torch +from cosmos1.models.tokenizer.inference.video_lib import CausalVideoTokenizer + +model_name = "Cosmos-0.1-Tokenizer-CV4x8x8" +input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16) # [B, C, T, H, W] +encoder = CausalVideoTokenizer(checkpoint_enc=f'checkpoints/{model_name}/encoder.jit') +(latent,) = encoder.encode(input_tensor) +torch.testing.assert_close(latent.shape, (1, 16, 3, 64, 64)) + +# The input tensor can be reconstructed by the decoder as: +decoder = CausalVideoTokenizer(checkpoint_dec=f'checkpoints/{model_name}/decoder.jit') +reconstructed_tensor = decoder.decode(latent) +torch.testing.assert_close(reconstructed_tensor.shape, input_tensor.shape) +``` +The `latent` will have the shape `(1, 16, 3, 64, 64)`, where the first of the three latents represents the first frame, and C=16 is the number of channels of the latent. + +### Encoding into Discrete Tokens +```python +import torch +from cosmos1.models.tokenizer.inference.video_lib import CausalVideoTokenizer + +model_name = "Cosmos-0.1-Tokenizer-DV4x8x8" +input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16) # [B, C, T, H, W] +encoder = CausalVideoTokenizer(checkpoint_enc=f'checkpoints/{model_name}/encoder.jit') +(indices, codes) = encoder.encode(input_tensor) +torch.testing.assert_close(indices.shape, (1, 3, 64, 64)) +torch.testing.assert_close(codes.shape, (1, 6, 3, 64, 64)) + +# The input tensor can be reconstructed by the decoder as: +decoder = CausalVideoTokenizer(checkpoint_dec=f'checkpoints/{model_name}/decoder.jit') +reconstructed_tensor = decoder.decode(indices) +torch.testing.assert_close(reconstructed_tensor.shape, input_tensor.shape) +``` +The `indices` will have the shape `(1, 3, 64, 64)` and contain integral values in the range `[1..64K]`, where the first of the three integral maps represents the first frame. +The `codes` will contain the pre-quantization continuous latent with shape `(1, 6, 3, 64, 64)`, where C=6 represents the number of FSQ levels. + +## Torchscript (PyTorch JIT) Inference APIs +The following instructions run the various tokenizer on the example image and video provided in `cosmos1/models/tokenizer/test_data/`. + +- Autoencoding images. Accepts an input image, and outputs a reconstruction of the image obtained by decoding the encoded latents. +```bash +# Autoencoding images using `Cosmos-CI` with a compression rate of 8x8. +model_name="Cosmos-0.1-Tokenizer-CI8x8" +python3 -m cosmos1.models.tokenizer.inference.image_cli \ + --image_pattern 'cosmos1/models/tokenizer/test_data/image.png' \ + --checkpoint_enc checkpoints/${model_name}/encoder.jit \ + --checkpoint_dec checkpoints/${model_name}/decoder.jit +``` +If `--output_dir` is not specified, you can find the reconstructed image at `cosmos1/models/tokenizer/test_data/reconstructions/image.png`. + +- Autoencoding videos. Accepts an input video, and outputs a reconstruction of the video obtained by decoding the encoded latents. +```bash +# Autoencoding videos using `Cosmos-DV` with a compression rate of 4x8x8. +model_name="Cosmos-0.1-Tokenizer-DV4x8x8" +python3 -m cosmos1.models.tokenizer.inference.video_cli \ + --video_pattern 'cosmos1/models/tokenizer/test_data/video.mp4' \ + --checkpoint_enc checkpoints/${model_name}/encoder.jit \ + --checkpoint_dec checkpoints/${model_name}/decoder.jit +``` +If `--output_dir` is not specified, then you can find the reconstructed video at `cosmos1/models/tokenizer/test_data/reconstructions/video.mp4`. + +## PyTorch Inference APIs + +To run the tokenizers in native PyTorch, append your commands with `--mode=torch`.
+In PyTorch mode, the model is constructed from the native network definition scripts, which requires providing additional arguments to configure the model for instantiation. + +For example, to instantiate a `Cosmos-DI` with a spatial compression factor of 8, append the following command line arguments: + +- `--mode=torch` +- `--tokenizer_type=DI8x8` + +Note that the `--checkpoint_enc`, `--checkpoint_dec`, and `--checkpoint` should still refer to JIT files.
+The necessary `state_dict`s will be extracted from the loaded JIT models to initialize the weights of the constructed native PyTorch model. + +```bash +# Autoencoding images using `Cosmos-DI` with a compression rate of 8x8. +model_name="Cosmos-0.1-Tokenizer-DI8x8" +python3 -m cosmos1.models.tokenizer.inference.image_cli \ + --image_pattern 'cosmos1/models/tokenizer/test_data/*.png' \ + --mode=torch \ + --tokenizer_type=DI8x8 \ + --checkpoint_enc checkpoints/${model_name}/encoder.jit \ + --checkpoint_dec checkpoints/${model_name}/decoder.jit +``` + +To instantiate a `Cosmos-CV` with a temporal factor of 8 and a spatial compression factor of 8, append the following command line arguments: + +- `--mode=torch` +- `--tokenizer_type=CV8x8x8` + +```bash +# Autoencoding videos using `Cosmos-CV` with a compression rate of 8x8x8. +model_name="Cosmos-1.0-Tokenizer-CV8x8x8" +python3 -m cosmos1.models.tokenizer.inference.video_cli \ + --video_pattern 'cosmos1/models/tokenizer/test_data/*.mp4' \ + --mode=torch \ + --tokenizer_type=CV8x8x8 \ + --checkpoint_enc checkpoints/${model_name}/encoder.jit \ + --checkpoint_dec checkpoints/${model_name}/decoder.jit +``` + +Similarly, to instantiate a `Cosmos-CV4x8x8-LowRes`, append the following command line arguments and the corresponding jit compiled ckpts: +- `--mode=torch` +- `--tokenizer_type=CV4x8x8-LowRes` + +```bash +# Autoencoding videos using `Cosmos-CV` with a compression rate of 8x8x8. +model_name="Cosmos-1.0-Tokenizer-CV4x8x8-LowRes" +python3 -m cosmos1.models.tokenizer.inference.video_cli \ + --video_pattern 'cosmos1/models/tokenizer/test_data/*.mp4' \ + --mode=torch \ + --tokenizer_type=CV4x8x8-LowRes \ + --checkpoint_enc checkpoints/${model_name}/encoder.jit \ + --checkpoint_dec checkpoints/${model_name}/decoder.jit +``` +## Inference & dataset tokenization with NeMo (JIT/TensorRT) +TensorRT inference is coming soon, which will be available in [Cosmos Tokenizer README within the NeMo repository](https://github.com/NVIDIA/NeMo/tree/main/nemo/collections/common/video_tokenizers) + +### JIT inference +Please install NeMo from the GitHub `main` branch following the instructions [here](https://github.com/NVIDIA/NeMo?tab=readme-ov-file#pip-from-a-source-branch). + +Run the following code to tokenize the video: + +```python +import torch +from nemo.collections.common.video_tokenizers.cosmos_vision_tokenizer import CausalVideoTokenizer +model_name = "Cosmos-0.1-Tokenizer-CV4x8x8" +model = CausalVideoTokenizer.from_pretrained(model_name) +input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16) +(latent, ) = model.encode(input_tensor) +``` + +### dataset tokenization and multimodal model training +Please see the [Cosmos Tokenizer README within the NeMo repository](https://github.com/NVIDIA/NeMo/tree/main/nemo/collections/common/video_tokenizers) for additional examples to create multimodal training datasets with the Cosmos Tokenizer. + + +## Evaluation +Quantitative comparision of our tokenizer and previous tokenizers on DAVIS (Perazzi et al., 2016) dataset. Cosmos Tokenizer achieves state-of-the-art results. Even at higer compression rates (8x8x8 and 8x16x16), Cosmos Tokenizer outperforms previous methods, demonstrating excellent compression-quality trade-off. +![Arch](assets/Davis-results.jpg) +## Performance +Comparision of parameter counts and average encoding and decoding times per image or per video frame on a single A100 80GB GPU. Cosmos Tokenizer achieves 2x to 12x faster speeds than previous methods while maintaining smallest model sizes, demonstrating high tokenization efficiency. +![Arch](assets/Performance.jpg) + + +## [TokenBench](https://github.com/NVlabs/TokenBench) +TokenBench is a comprehensive benchmark that we have curated to standardize the evaluation of [Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos/blob/main/cosmos1/models/tokenizer). It covers a wide variety of domains including robotic manipulation, driving, egocentric, and web videos. It consists of high-resolution, long-duration videos, and is designed to benchmark video tokenizers. We have made TokenBench publicly available at [github.com/NVlabs/TokenBench](https://github.com/NVlabs/TokenBench). + +## Core Contributors + +Fitsum Reda, Jinwei Gu, Xian Liu, Songwei Ge, Ting-Chun Wang, Haoxiang Wang, Ming-Yu Liu + + +## Citation + +If you find Cosmos Tokenizer useful in your works, please acknowledge it +appropriately by citing: + +``` +@article{agarwal2025cosmos, + title={Cosmos World Foundation Model Platform for Physical AI}, + author={NVIDIA et. al.}, + journal={arXiv preprint arXiv:2501.03575}, + year={2025} +} +``` +## Acknowledgments +We would like to acknowledge the following projects where parts of the codes in the [cosmos1/models/tokenizer/modules](cosmos1/models/tokenizer/modules) folder is derived from: +- [CompVis/stable-diffusion](https://github.com/CompVis/stable-diffusion) +- [lucidrains/magvit2-pytorch](https://github.com/lucidrains/magvit2-pytorch) +- [lucidrains/vector-quantize-pytorch](https://github.com/lucidrains/vector-quantize-pytorch) +- [CompVis/taming-transformers](https://github.com/CompVis/taming-transformers) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/Davis-results.jpg b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/Davis-results.jpg new file mode 100644 index 00000000..6b3d1cf6 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/Davis-results.jpg differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/Performance.jpg b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/Performance.jpg new file mode 100644 index 00000000..5d81c281 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/Performance.jpg differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/arch_diagram.jpg b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/arch_diagram.jpg new file mode 100644 index 00000000..61d3fb9a Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/arch_diagram.jpg differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/cosmos-tokenizer.jpg b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/cosmos-tokenizer.jpg new file mode 100644 index 00000000..3e283d51 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/assets/cosmos-tokenizer.jpg differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/__init__.py new file mode 100644 index 00000000..3159bfe6 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/image_cli.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/image_cli.py new file mode 100644 index 00000000..19cc08ac --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/image_cli.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""A CLI to run ImageTokenizer on plain images based on torch.jit. + +Usage: + python3 -m cosmos1.models.tokenizer.inference.image_cli \ + --image_pattern 'path/to/input/folder/*.jpg' \ + --output_dir ./reconstructions \ + --checkpoint_enc ./checkpoints//encoder.jit \ + --checkpoint_dec ./checkpoints//decoder.jit + + Optionally, you can run the model in pure PyTorch mode: + python3 -m cosmos1.models.tokenizer.inference.image_cli \ + --image_pattern 'path/to/input/folder/*.jpg' \ + --mode torch \ + --tokenizer_type CI8x8 \ + --checkpoint_enc ./checkpoints//encoder.jit \ + --checkpoint_dec ./checkpoints//decoder.jit +""" + +import os +import sys +from argparse import ArgumentParser, Namespace +from typing import Any + +import numpy as np +from cosmos1.models.tokenizer.inference.image_lib import ImageTokenizer +from cosmos1.models.tokenizer.inference.utils import ( + get_filepaths, + get_output_filepath, + read_image, + resize_image, + write_image, +) +from cosmos1.models.tokenizer.networks import TokenizerConfigs +from loguru import logger as logging + + +def _parse_args() -> tuple[Namespace, dict[str, Any]]: + parser = ArgumentParser(description="A CLI for running ImageTokenizer on plain images.") + parser.add_argument( + "--image_pattern", + type=str, + default="path/to/images/*.jpg", + help="Glob pattern.", + ) + parser.add_argument( + "--checkpoint", + type=str, + default=None, + help="JIT full Autoencoder model filepath.", + ) + parser.add_argument( + "--checkpoint_enc", + type=str, + default=None, + help="JIT Encoder model filepath.", + ) + parser.add_argument( + "--checkpoint_dec", + type=str, + default=None, + help="JIT Decoder model filepath.", + ) + parser.add_argument( + "--tokenizer_type", + type=str, + default=None, + choices=[ + "CI8x8", + "DI8x8", + "CI16x16", + "DI16x16", + "CI8x8-LowRes", + "CI16x16-LowRes", + "DI8x8-LowRes", + "DI16x16-LowRes", + ], + help="Specifies the tokenizer type.", + ) + parser.add_argument( + "--mode", + type=str, + choices=["torch", "jit"], + default="jit", + help="Specify the backend: native 'torch' or 'jit' (default: 'jit')", + ) + parser.add_argument( + "--short_size", + type=int, + default=None, + help="The size to resample inputs. None, by default.", + ) + parser.add_argument( + "--dtype", + type=str, + default="bfloat16", + help="Sets the precision. Default bfloat16.", + ) + parser.add_argument( + "--device", + type=str, + default="cuda", + help="Device for invoking the model.", + ) + parser.add_argument("--output_dir", type=str, default=None, help="Output directory.") + parser.add_argument( + "--save_input", + action="store_true", + help="If on, the input image will be be outputed too.", + ) + args = parser.parse_args() + return args + + +logging.info("Initializes args ...") +args = _parse_args() +if args.mode == "torch" and args.tokenizer_type is None: + logging.error("'torch' backend requires the tokenizer_type to be specified.") + sys.exit(1) + + +def _run_eval() -> None: + """Invokes the evaluation pipeline.""" + + if args.checkpoint_enc is None and args.checkpoint_dec is None and args.checkpoint is None: + logging.warning("Aborting. Both encoder or decoder JIT required. Or provide the full autoencoder JIT model.") + return + + if args.mode == "torch": + _type = args.tokenizer_type.replace("-", "_") + _config = TokenizerConfigs[_type].value + else: + _config = None + + logging.info( + f"Loading a torch.jit model `{os.path.dirname(args.checkpoint or args.checkpoint_enc or args.checkpoint_dec)}` ..." + ) + autoencoder = ImageTokenizer( + checkpoint=args.checkpoint, + checkpoint_enc=args.checkpoint_enc, + checkpoint_dec=args.checkpoint_dec, + tokenizer_config=_config, + device=args.device, + dtype=args.dtype, + ) + + filepaths = get_filepaths(args.image_pattern) + logging.info(f"Found {len(filepaths)} images from {args.image_pattern}.") + + for filepath in filepaths: + logging.info(f"Reading image {filepath} ...") + image = read_image(filepath) + image = resize_image(image, short_size=args.short_size) + batch_image = np.expand_dims(image, axis=0) + + logging.info("Invoking the autoencoder model in ... ") + output_image = autoencoder(batch_image)[0] + + output_filepath = get_output_filepath(filepath, output_dir=args.output_dir) + logging.info(f"Outputing {output_filepath} ...") + write_image(output_filepath, output_image) + + if args.save_input: + ext = os.path.splitext(output_filepath)[-1] + input_filepath = output_filepath.replace(ext, "_input" + ext) + write_image(input_filepath, image) + + +@logging.catch(reraise=True) +def main() -> None: + _run_eval() + + +if __name__ == "__main__": + main() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/image_lib.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/image_lib.py new file mode 100644 index 00000000..52ed8146 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/image_lib.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""A library for image tokenizers inference.""" + +from typing import Any + +import numpy as np +import torch +from cosmos1.models.tokenizer.inference.utils import ( + load_decoder_model, + load_encoder_model, + load_model, + numpy2tensor, + pad_image_batch, + tensor2numpy, + unpad_image_batch, +) + + +class ImageTokenizer(torch.nn.Module): + def __init__( + self, + checkpoint: str = None, + checkpoint_enc: str = None, + checkpoint_dec: str = None, + tokenizer_config: dict[str, Any] = None, + device: str = "cuda", + dtype: str = "bfloat16", + ) -> None: + super().__init__() + self._device = device + self._dtype = getattr(torch, dtype) + self._full_model = ( + load_model(checkpoint, tokenizer_config, device).to(self._dtype) if checkpoint is not None else None + ) + self._enc_model = ( + load_encoder_model(checkpoint_enc, tokenizer_config, device).to(self._dtype) + if checkpoint_enc is not None + else None + ) + self._dec_model = ( + load_decoder_model(checkpoint_dec, tokenizer_config, device).to(self._dtype) + if checkpoint_dec is not None + else None + ) + + @torch.no_grad() + def autoencode(self, input_tensor: torch.Tensor) -> torch.Tensor: + """Reconstrcuts a batch of image tensors after embedding into a latent. + + Args: + input_tensor: The input image Bx3xHxW layout, range [-1..1]. + Returns: + The reconstructed tensor, layout Bx3xHxW, range [-1..1]. + """ + if self._full_model is not None: + output_tensor = self._full_model(input_tensor) + output_tensor = output_tensor[0] if isinstance(output_tensor, tuple) else output_tensor + else: + output_latent = self.encode(input_tensor)[0] + output_tensor = self.decode(output_latent) + return output_tensor + + @torch.no_grad() + def decode(self, input_latent: torch.Tensor) -> torch.Tensor: + """Decodes an image from a provided latent embedding. + + Args: + input_latent: The continuous latent Bx16xhxw for CI, + or the discrete indices Bxhxw for DI. + Returns: + The output tensor in Bx3xHxW, range [-1..1]. + """ + return self._dec_model(input_latent) + + @torch.no_grad() + def encode(self, input_tensor: torch.Tensor) -> tuple[torch.Tensor]: + """Encodes an image into a latent embedding or code. + + Args: + input_tensor: The input tensor Bx3xHxW layout, range [-1..1]. + Returns: + For continuous image (CI) tokenizer, the tuple contains: + - The latent embedding, Bx16x(h)x(w), where the compression + rate is (H/h x W/w), and channel dimension of 16. + For discrete image (DI) tokenizer, the tuple contains: + - The indices, Bx(h)x(w), from a codebook of size 64K, which + corresponds to FSQ levels of (8,8,8,5,5,5). + - The discrete code, Bx6x(h)x(w), where the compression rate is + again (H/h x W/w), and channel dimension of 6. + """ + output_latent = self._enc_model(input_tensor) + if isinstance(output_latent, torch.Tensor): + return output_latent + return output_latent[:-1] + + @torch.no_grad() + def forward(self, image: np.ndarray) -> np.ndarray: + """Reconstructs an image using a pre-trained tokenizer. + + Args: + image: The input image BxHxWxC layout, range [0..255]. + Returns: + The reconstructed image in range [0..255], layout BxHxWxC. + """ + padded_input_image, crop_region = pad_image_batch(image) + input_tensor = numpy2tensor(padded_input_image, dtype=self._dtype, device=self._device) + output_tensor = self.autoencode(input_tensor) + padded_output_image = tensor2numpy(output_tensor) + return unpad_image_batch(padded_output_image, crop_region) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/utils.py new file mode 100644 index 00000000..c7c2fc54 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/utils.py @@ -0,0 +1,402 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility functions for the inference libraries.""" + +import os +from glob import glob +from typing import Any + +import mediapy as media +import numpy as np +import torch +from cosmos1.models.tokenizer.networks import TokenizerModels + + +_DTYPE, _DEVICE = torch.bfloat16, "cuda" +_UINT8_MAX_F = float(torch.iinfo(torch.uint8).max) +_SPATIAL_ALIGN = 16 +_TEMPORAL_ALIGN = 8 + + +def load_model( + jit_filepath: str = None, + tokenizer_config: dict[str, Any] = None, + device: str = "cuda", +) -> torch.nn.Module | torch.jit.ScriptModule: + """Loads a torch.nn.Module from a filepath. + + Args: + jit_filepath: The filepath to the JIT-compiled model. + device: The device to load the model onto, default=cuda. + Returns: + The JIT compiled model loaded to device and on eval mode. + """ + if tokenizer_config is None: + return load_jit_model(jit_filepath, device) + full_model, ckpts = _load_pytorch_model(jit_filepath, tokenizer_config, device) + full_model.load_state_dict(ckpts.state_dict(), strict=True) + return full_model.eval().to(device) + + +def load_encoder_model( + jit_filepath: str = None, + tokenizer_config: dict[str, Any] = None, + device: str = "cuda", +) -> torch.nn.Module | torch.jit.ScriptModule: + """Loads a torch.nn.Module from a filepath. + + Args: + jit_filepath: The filepath to the JIT-compiled model. + device: The device to load the model onto, default=cuda. + Returns: + The JIT compiled model loaded to device and on eval mode. + """ + if tokenizer_config is None: + return load_jit_model(jit_filepath, device) + full_model, ckpts = _load_pytorch_model(jit_filepath, tokenizer_config, device) + encoder_model = full_model.encoder_jit() + encoder_model.load_state_dict(ckpts.state_dict(), strict=True) + return encoder_model.eval().to(device) + + +def load_decoder_model( + jit_filepath: str = None, + tokenizer_config: dict[str, Any] = None, + device: str = "cuda", +) -> torch.nn.Module | torch.jit.ScriptModule: + """Loads a torch.nn.Module from a filepath. + + Args: + jit_filepath: The filepath to the JIT-compiled model. + device: The device to load the model onto, default=cuda. + Returns: + The JIT compiled model loaded to device and on eval mode. + """ + if tokenizer_config is None: + return load_jit_model(jit_filepath, device) + full_model, ckpts = _load_pytorch_model(jit_filepath, tokenizer_config, device) + decoder_model = full_model.decoder_jit() + decoder_model.load_state_dict(ckpts.state_dict(), strict=True) + return decoder_model.eval().to(device) + + +def _load_pytorch_model( + jit_filepath: str = None, tokenizer_config: str = None, device: str = "cuda" +) -> torch.nn.Module: + """Loads a torch.nn.Module from a filepath. + + Args: + jit_filepath: The filepath to the JIT-compiled model. + device: The device to load the model onto, default=cuda. + Returns: + The JIT compiled model loaded to device and on eval mode. + """ + tokenizer_name = tokenizer_config["name"] + model = TokenizerModels[tokenizer_name].value(**tokenizer_config) + ckpts = torch.jit.load(jit_filepath, map_location=device) + return model, ckpts + + +def load_jit_model(jit_filepath: str = None, device: str = "cuda") -> torch.jit.ScriptModule: + """Loads a torch.jit.ScriptModule from a filepath. + + Args: + jit_filepath: The filepath to the JIT-compiled model. + device: The device to load the model onto, default=cuda. + Returns: + The JIT compiled model loaded to device and on eval mode. + """ + model = torch.jit.load(jit_filepath, map_location=device) + return model.eval().to(device) + + +def save_jit_model( + model: torch.jit.ScriptModule | torch.jit.RecursiveScriptModule = None, + jit_filepath: str = None, +) -> None: + """Saves a torch.jit.ScriptModule or torch.jit.RecursiveScriptModule to file. + + Args: + model: JIT compiled model loaded onto `config.checkpoint.jit.device`. + jit_filepath: The filepath to the JIT-compiled model. + """ + torch.jit.save(model, jit_filepath) + + +def get_filepaths(input_pattern) -> list[str]: + """Returns a list of filepaths from a pattern.""" + filepaths = sorted(glob(str(input_pattern))) + return list(set(filepaths)) + + +def get_output_filepath(filepath: str, output_dir: str = None) -> str: + """Returns the output filepath for the given input filepath.""" + output_dir = output_dir or f"{os.path.dirname(filepath)}/reconstructions" + output_filepath = f"{output_dir}/{os.path.basename(filepath)}" + os.makedirs(output_dir, exist_ok=True) + return output_filepath + + +def read_image(filepath: str) -> np.ndarray: + """Reads an image from a filepath. + + Args: + filepath: The filepath to the image. + + Returns: + The image as a numpy array, layout HxWxC, range [0..255], uint8 dtype. + """ + image = media.read_image(filepath) + # convert the grey scale image to RGB + # since our tokenizers always assume 3-channel RGB image + if image.ndim == 2: + image = np.stack([image] * 3, axis=-1) + # convert RGBA to RGB + if image.shape[-1] == 4: + image = image[..., :3] + return image + + +def read_video(filepath: str) -> np.ndarray: + """Reads a video from a filepath. + + Args: + filepath: The filepath to the video. + Returns: + The video as a numpy array, layout TxHxWxC, range [0..255], uint8 dtype. + """ + video = media.read_video(filepath) + # convert the grey scale frame to RGB + # since our tokenizers always assume 3-channel video + if video.ndim == 3: + video = np.stack([video] * 3, axis=-1) + # convert RGBA to RGB + if video.shape[-1] == 4: + video = video[..., :3] + return video + + +def resize_image(image: np.ndarray, short_size: int = None) -> np.ndarray: + """Resizes an image to have the short side of `short_size`. + + Args: + image: The image to resize, layout HxWxC, of any range. + short_size: The size of the short side. + Returns: + The resized image. + """ + if short_size is None: + return image + height, width = image.shape[-3:-1] + if height <= width: + height_new, width_new = short_size, int(width * short_size / height + 0.5) + width_new = width_new if width_new % 2 == 0 else width_new + 1 + else: + height_new, width_new = ( + int(height * short_size / width + 0.5), + short_size, + ) + height_new = height_new if height_new % 2 == 0 else height_new + 1 + return media.resize_image(image, shape=(height_new, width_new)) + + +def resize_video(video: np.ndarray, short_size: int = None) -> np.ndarray: + """Resizes a video to have the short side of `short_size`. + + Args: + video: The video to resize, layout TxHxWxC, of any range. + short_size: The size of the short side. + Returns: + The resized video. + """ + if short_size is None: + return video + height, width = video.shape[-3:-1] + if height <= width: + height_new, width_new = short_size, int(width * short_size / height + 0.5) + width_new = width_new if width_new % 2 == 0 else width_new + 1 + else: + height_new, width_new = ( + int(height * short_size / width + 0.5), + short_size, + ) + height_new = height_new if height_new % 2 == 0 else height_new + 1 + return media.resize_video(video, shape=(height_new, width_new)) + + +def write_image(filepath: str, image: np.ndarray): + """Writes an image to a filepath.""" + return media.write_image(filepath, image) + + +def write_video(filepath: str, video: np.ndarray, fps: int = 24) -> None: + """Writes a video to a filepath.""" + return media.write_video(filepath, video, fps=fps) + + +def numpy2tensor( + input_image: np.ndarray, + dtype: torch.dtype = _DTYPE, + device: str = _DEVICE, + range_min: int = -1, +) -> torch.Tensor: + """Converts image(dtype=np.uint8) to `dtype` in range [0..255]. + + Args: + input_image: A batch of images in range [0..255], BxHxWx3 layout. + Returns: + A torch.Tensor of layout Bx3xHxW in range [-1..1], dtype. + """ + ndim = input_image.ndim + indices = list(range(1, ndim))[-1:] + list(range(1, ndim))[:-1] + image = input_image.transpose((0,) + tuple(indices)) / _UINT8_MAX_F + if range_min == -1: + image = 2.0 * image - 1.0 + return torch.from_numpy(image).to(dtype).to(device) + + +def tensor2numpy(input_tensor: torch.Tensor, range_min: int = -1) -> np.ndarray: + """Converts tensor in [-1,1] to image(dtype=np.uint8) in range [0..255]. + + Args: + input_tensor: Input image tensor of Bx3xHxW layout, range [-1..1]. + Returns: + A numpy image of layout BxHxWx3, range [0..255], uint8 dtype. + """ + if range_min == -1: + input_tensor = (input_tensor.float() + 1.0) / 2.0 + ndim = input_tensor.ndim + output_image = input_tensor.clamp(0, 1).cpu().numpy() + output_image = output_image.transpose((0,) + tuple(range(2, ndim)) + (1,)) + return (output_image * _UINT8_MAX_F + 0.5).astype(np.uint8) + + +def pad_image_batch(batch: np.ndarray, spatial_align: int = _SPATIAL_ALIGN) -> tuple[np.ndarray, list[int]]: + """Pads a batch of images to be divisible by `spatial_align`. + + Args: + batch: The batch of images to pad, layout BxHxWx3, in any range. + align: The alignment to pad to. + Returns: + The padded batch and the crop region. + """ + height, width = batch.shape[1:3] + align = spatial_align + height_to_pad = (align - height % align) if height % align != 0 else 0 + width_to_pad = (align - width % align) if width % align != 0 else 0 + + crop_region = [ + height_to_pad >> 1, + width_to_pad >> 1, + height + (height_to_pad >> 1), + width + (width_to_pad >> 1), + ] + batch = np.pad( + batch, + ( + (0, 0), + (height_to_pad >> 1, height_to_pad - (height_to_pad >> 1)), + (width_to_pad >> 1, width_to_pad - (width_to_pad >> 1)), + (0, 0), + ), + mode="constant", + ) + return batch, crop_region + + +def pad_video_batch( + batch: np.ndarray, + temporal_align: int = _TEMPORAL_ALIGN, + spatial_align: int = _SPATIAL_ALIGN, +) -> tuple[np.ndarray, list[int]]: + """Pads a batch of videos to be divisible by `temporal_align` or `spatial_align`. + + Zero pad spatially. Reflection pad temporally to handle causality better. + Args: + batch: The batch of videos to pad., layout BxFxHxWx3, in any range. + align: The alignment to pad to. + Returns: + The padded batch and the crop region. + """ + num_frames, height, width = batch.shape[-4:-1] + align = spatial_align + height_to_pad = (align - height % align) if height % align != 0 else 0 + width_to_pad = (align - width % align) if width % align != 0 else 0 + + align = temporal_align + frames_to_pad = (align - (num_frames - 1) % align) if (num_frames - 1) % align != 0 else 0 + + crop_region = [ + frames_to_pad >> 1, + height_to_pad >> 1, + width_to_pad >> 1, + num_frames + (frames_to_pad >> 1), + height + (height_to_pad >> 1), + width + (width_to_pad >> 1), + ] + batch = np.pad( + batch, + ( + (0, 0), + (0, 0), + (height_to_pad >> 1, height_to_pad - (height_to_pad >> 1)), + (width_to_pad >> 1, width_to_pad - (width_to_pad >> 1)), + (0, 0), + ), + mode="constant", + ) + batch = np.pad( + batch, + ( + (0, 0), + (frames_to_pad >> 1, frames_to_pad - (frames_to_pad >> 1)), + (0, 0), + (0, 0), + (0, 0), + ), + mode="edge", + ) + return batch, crop_region + + +def unpad_video_batch(batch: np.ndarray, crop_region: list[int]) -> np.ndarray: + """Unpads video with `crop_region`. + + Args: + batch: A batch of numpy videos, layout BxFxHxWxC. + crop_region: [f1,y1,x1,f2,y2,x2] first, top, left, last, bot, right crop indices. + + Returns: + np.ndarray: Cropped numpy video, layout BxFxHxWxC. + """ + assert len(crop_region) == 6, "crop_region should be len of 6." + f1, y1, x1, f2, y2, x2 = crop_region + return batch[..., f1:f2, y1:y2, x1:x2, :] + + +def unpad_image_batch(batch: np.ndarray, crop_region: list[int]) -> np.ndarray: + """Unpads image with `crop_region`. + + Args: + batch: A batch of numpy images, layout BxHxWxC. + crop_region: [y1,x1,y2,x2] top, left, bot, right crop indices. + + Returns: + np.ndarray: Cropped numpy image, layout BxHxWxC. + """ + assert len(crop_region) == 4, "crop_region should be len of 4." + y1, x1, y2, x2 = crop_region + return batch[..., y1:y2, x1:x2, :] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/video_cli.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/video_cli.py new file mode 100644 index 00000000..aff4d231 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/video_cli.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""A CLI to run CausalVideoTokenizer on plain videos based on torch.jit. + +Usage: + python3 -m cosmos1.models.tokenizer.inference.video_cli \ + --video_pattern 'path/to/video/samples/*.mp4' \ + --output_dir ./reconstructions \ + --checkpoint_enc ./checkpoints//encoder.jit \ + --checkpoint_dec ./checkpoints//decoder.jit + + Optionally, you can run the model in pure PyTorch mode: + python3 -m cosmos1.models.tokenizer.inference.video_cli \ + --video_pattern 'path/to/video/samples/*.mp4' \ + --mode=torch \ + --tokenizer_type=CV \ + --temporal_compression=4 \ + --spatial_compression=8 \ + --checkpoint_enc ./checkpoints//encoder.jit \ + --checkpoint_dec ./checkpoints//decoder.jit +""" + +import os +import sys +from argparse import ArgumentParser, Namespace +from typing import Any + +import numpy as np +from cosmos1.models.tokenizer.inference.utils import ( + get_filepaths, + get_output_filepath, + read_video, + resize_video, + write_video, +) +from cosmos1.models.tokenizer.inference.video_lib import CausalVideoTokenizer +from cosmos1.models.tokenizer.networks import TokenizerConfigs +from loguru import logger as logging + + +def _parse_args() -> tuple[Namespace, dict[str, Any]]: + parser = ArgumentParser(description="A CLI for CausalVideoTokenizer.") + parser.add_argument( + "--video_pattern", + type=str, + default="path/to/videos/*.mp4", + help="Glob pattern.", + ) + parser.add_argument( + "--checkpoint", + type=str, + default=None, + help="JIT full Autoencoder model filepath.", + ) + parser.add_argument( + "--checkpoint_enc", + type=str, + default=None, + help="JIT Encoder model filepath.", + ) + parser.add_argument( + "--checkpoint_dec", + type=str, + default=None, + help="JIT Decoder model filepath.", + ) + parser.add_argument( + "--tokenizer_type", + type=str, + default=None, + choices=[ + "CV4x8x8", + "DV4x8x8", + "CV8x8x8", + "DV8x8x8", + "CV8x16x16", + "DV8x16x16", + "CV4x8x8-LowRes", + "DV4x8x8-LowRes", + ], + help="Specifies the tokenizer type.", + ) + parser.add_argument( + "--mode", + type=str, + choices=["torch", "jit"], + default="jit", + help="Specify the backend: native 'torch' or 'jit' (default: 'jit')", + ) + parser.add_argument( + "--short_size", + type=int, + default=None, + help="The size to resample inputs. None, by default.", + ) + parser.add_argument( + "--temporal_window", + type=int, + default=17, + help="The temporal window to operate at a time.", + ) + parser.add_argument( + "--dtype", + type=str, + default="bfloat16", + help="Sets the precision, default bfloat16.", + ) + parser.add_argument( + "--device", + type=str, + default="cuda", + help="Device for invoking the model.", + ) + parser.add_argument("--output_dir", type=str, default=None, help="Output directory.") + parser.add_argument( + "--output_fps", + type=float, + default=24.0, + help="Output frames-per-second (FPS).", + ) + parser.add_argument( + "--save_input", + action="store_true", + help="If on, the input video will be be outputted too.", + ) + args = parser.parse_args() + return args + + +logging.info("Initializes args ...") +args = _parse_args() +if args.mode == "torch" and args.tokenizer_type is None: + logging.error("`torch` backend requires `--tokenizer_type` to be specified.") + sys.exit(1) + + +def _run_eval() -> None: + """Invokes JIT-compiled CausalVideoTokenizer on an input video.""" + + if args.checkpoint_enc is None and args.checkpoint_dec is None and args.checkpoint is None: + logging.warning("Aborting. Both encoder or decoder JIT required. Or provide the full autoencoder JIT model.") + return + + if args.mode == "torch": + _type = args.tokenizer_type.replace("-", "_") + _config = TokenizerConfigs[_type].value + else: + _config = None + + logging.info( + f"Loading a torch.jit model `{os.path.dirname(args.checkpoint or args.checkpoint_enc or args.checkpoint_dec)}` ..." + ) + autoencoder = CausalVideoTokenizer( + checkpoint=args.checkpoint, + checkpoint_enc=args.checkpoint_enc, + checkpoint_dec=args.checkpoint_dec, + tokenizer_config=_config, + device=args.device, + dtype=args.dtype, + ) + + logging.info(f"Looking for files matching video_pattern={args.video_pattern} ...") + filepaths = get_filepaths(args.video_pattern) + logging.info(f"Found {len(filepaths)} videos from {args.video_pattern}.") + + for filepath in filepaths: + logging.info(f"Reading video {filepath} ...") + video = read_video(filepath) + video = resize_video(video, short_size=args.short_size) + + logging.info("Invoking the autoencoder model in ... ") + batch_video = video[np.newaxis, ...] + output_video = autoencoder(batch_video, temporal_window=args.temporal_window)[0] + logging.info("Constructing output filepath ...") + output_filepath = get_output_filepath(filepath, output_dir=args.output_dir) + logging.info(f"Outputing {output_filepath} ...") + write_video(output_filepath, output_video, fps=args.output_fps) + if args.save_input: + ext = os.path.splitext(output_filepath)[-1] + input_filepath = output_filepath.replace(ext, "_input" + ext) + write_video(input_filepath, video, fps=args.output_fps) + + +@logging.catch(reraise=True) +def main() -> None: + _run_eval() + + +if __name__ == "__main__": + main() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/video_lib.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/video_lib.py new file mode 100644 index 00000000..183d2fbe --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/inference/video_lib.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""A library for Causal Video Tokenizer inference.""" + +from typing import Any + +import numpy as np +import torch +from cosmos1.models.tokenizer.inference.utils import ( + load_decoder_model, + load_encoder_model, + load_model, + numpy2tensor, + pad_video_batch, + tensor2numpy, + unpad_video_batch, +) +from tqdm import tqdm + + +class CausalVideoTokenizer(torch.nn.Module): + def __init__( + self, + checkpoint: str = None, + checkpoint_enc: str = None, + checkpoint_dec: str = None, + tokenizer_config: dict[str, Any] = None, + device: str = "cuda", + dtype: str = "bfloat16", + ) -> None: + super().__init__() + self._device = device + self._dtype = getattr(torch, dtype) + self._full_model = ( + load_model(checkpoint, tokenizer_config, device).to(self._dtype) if checkpoint is not None else None + ) + self._enc_model = ( + load_encoder_model(checkpoint_enc, tokenizer_config, device).to(self._dtype) + if checkpoint_enc is not None + else None + ) + self._dec_model = ( + load_decoder_model(checkpoint_dec, tokenizer_config, device).to(self._dtype) + if checkpoint_dec is not None + else None + ) + + @torch.no_grad() + def autoencode(self, input_tensor: torch.Tensor) -> torch.Tensor: + """Reconstrcuts a batch of video tensors after embedding into a latent. + + Args: + video: The input video Bx3xTxHxW layout, range [-1..1]. + Returns: + The reconstructed video, layout Bx3xTxHxW, range [-1..1]. + """ + if self._full_model is not None: + output_tensor = self._full_model(input_tensor) + output_tensor = output_tensor[0] if isinstance(output_tensor, tuple) else output_tensor + else: + output_latent = self.encode(input_tensor)[0] + output_tensor = self.decode(output_latent) + return output_tensor + + @torch.no_grad() + def encode(self, input_tensor: torch.Tensor) -> tuple[torch.Tensor]: + """Encodes a numpy video into a CausalVideo latent or code. + + Args: + input_tensor: The input tensor Bx3xTxHxW layout, range [-1..1]. + Returns: + For causal continuous video (CV) tokenizer, the tuple contains: + - The latent embedding, Bx16x(t)x(h)x(w), where the compression + rate is (T/t x H/h x W/w), and channel dimension of 16. + For causal discrete video (DV) tokenizer, the tuple contains: + 1) The indices, Bx(t)x(h)x(w), from a codebook of size 64K, which + is formed by FSQ levels of (8,8,8,5,5,5). + 2) The discrete code, Bx6x(t)x(h)x(w), where the compression rate + is again (T/t x H/h x W/w), and channel dimension of 6. + """ + assert input_tensor.ndim == 5, "input video should be of 5D." + + output_latent = self._enc_model(input_tensor) + if isinstance(output_latent, torch.Tensor): + return output_latent + return output_latent[:-1] + + @torch.no_grad() + def decode(self, input_latent: torch.Tensor) -> torch.Tensor: + """Encodes a numpy video into a CausalVideo latent. + + Args: + input_latent: The continuous latent Bx16xtxhxw for CV, + or the discrete indices Bxtxhxw for DV. + Returns: + The reconstructed tensor, layout [B,3,1+(T-1)*8,H*16,W*16] in range [-1..1]. + """ + assert input_latent.ndim >= 4, "input latent should be of 5D for continuous and 4D for discrete." + return self._dec_model(input_latent) + + def forward( + self, + video: np.ndarray, + temporal_window: int = 17, + ) -> np.ndarray: + """Reconstructs video using a pre-trained CausalTokenizer autoencoder. + Given a video of arbitrary length, the forward invokes the CausalVideoTokenizer + in a sliding manner with a `temporal_window` size. + + Args: + video: The input video BxTxHxWx3 layout, range [0..255]. + temporal_window: The length of the temporal window to process, default=25. + Returns: + The reconstructed video in range [0..255], layout BxTxHxWx3. + """ + assert video.ndim == 5, "input video should be of 5D." + num_frames = video.shape[1] # can be of any length. + output_video_list = [] + for idx in tqdm(range(0, (num_frames - 1) // temporal_window + 1)): + # Input video for the current window. + start, end = idx * temporal_window, (idx + 1) * temporal_window + input_video = video[:, start:end, ...] + + # Spatio-temporally pad input_video so it's evenly divisible. + padded_input_video, crop_region = pad_video_batch(input_video) + input_tensor = numpy2tensor(padded_input_video, dtype=self._dtype, device=self._device) + output_tensor = self.autoencode(input_tensor) + padded_output_video = tensor2numpy(output_tensor) + output_video = unpad_video_batch(padded_output_video, crop_region) + + output_video_list.append(output_video) + return np.concatenate(output_video_list, axis=1) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/__init__.py new file mode 100644 index 00000000..8ea75495 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/__init__.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from enum import Enum + +from cosmos1.models.tokenizer.modules.distributions import GaussianDistribution, IdentityDistribution +from cosmos1.models.tokenizer.modules.layers2d import Decoder, Encoder +from cosmos1.models.tokenizer.modules.layers3d import DecoderBase, DecoderFactorized, EncoderBase, EncoderFactorized +from cosmos1.models.tokenizer.modules.quantizers import FSQuantizer, LFQuantizer, ResidualFSQuantizer, VectorQuantizer + + +class EncoderType(Enum): + Default = Encoder + + +class DecoderType(Enum): + Default = Decoder + + +class Encoder3DType(Enum): + BASE = EncoderBase + FACTORIZED = EncoderFactorized + + +class Decoder3DType(Enum): + BASE = DecoderBase + FACTORIZED = DecoderFactorized + + +class ContinuousFormulation(Enum): + VAE = GaussianDistribution + AE = IdentityDistribution + + +class DiscreteQuantizer(Enum): + VQ = VectorQuantizer + LFQ = LFQuantizer + FSQ = FSQuantizer + RESFSQ = ResidualFSQuantizer diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/distributions.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/distributions.py new file mode 100644 index 00000000..c90035a7 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/distributions.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The distribution modes to use for continuous image tokenizers.""" + +import torch + + +class IdentityDistribution(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self, parameters): + return parameters, (torch.tensor([0.0]), torch.tensor([0.0])) + + +class GaussianDistribution(torch.nn.Module): + def __init__(self, min_logvar: float = -30.0, max_logvar: float = 20.0): + super().__init__() + self.min_logvar = min_logvar + self.max_logvar = max_logvar + + def sample(self, mean, logvar): + std = torch.exp(0.5 * logvar) + return mean + std * torch.randn_like(mean) + + def forward(self, parameters): + mean, logvar = torch.chunk(parameters, 2, dim=1) + logvar = torch.clamp(logvar, self.min_logvar, self.max_logvar) + return self.sample(mean, logvar), (mean, logvar) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers2d.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers2d.py new file mode 100644 index 00000000..6c5cc13b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers2d.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The model definition for Continuous 2D layers + +Adapted from: https://github.com/CompVis/stable-diffusion/blob/ +21f890f9da3cfbeaba8e2ac3c425ee9e998d5229/ldm/modules/diffusionmodules/model.py + +[Copyright (c) 2022 Robin Rombach and Patrick Esser and contributors] +https://github.com/CompVis/stable-diffusion/blob/ +21f890f9da3cfbeaba8e2ac3c425ee9e998d5229/LICENSE +""" + +import math + +import numpy as np + +# pytorch_diffusion + derived encoder decoder +import torch +import torch.nn as nn +import torch.nn.functional as F +from cosmos1.models.tokenizer.modules.patching import Patcher, UnPatcher +from cosmos1.models.tokenizer.modules.utils import Normalize, nonlinearity +from loguru import logger as logging + + +class Upsample(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.repeat_interleave(2, dim=2).repeat_interleave(2, dim=3) + return self.conv(x) + + +class Downsample(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad = (0, 1, 0, 1) + x = F.pad(x, pad, mode="constant", value=0) + return self.conv(x) + + +class ResnetBlock(nn.Module): + def __init__( + self, + *, + in_channels: int, + out_channels: int = None, + dropout: float, + **kwargs, + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + + self.norm1 = Normalize(in_channels) + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = Normalize(out_channels) + self.dropout = nn.Dropout(dropout) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.nin_shortcut = ( + nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + if in_channels != out_channels + else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + x = self.nin_shortcut(x) + + return x + h + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int): + super().__init__() + + self.norm = Normalize(in_channels) + self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # TODO (freda): Consider reusing implementations in Attn `imaginaire`, + # since than one is gonna be based on TransformerEngine's attn op, + # w/c could ease CP implementations. + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) + k = k.reshape(b, c, h * w) + w_ = torch.bmm(q, k) + w_ = w_ * (int(c) ** (-0.5)) + w_ = F.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) + h_ = torch.bmm(v, w_) + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +class Encoder(nn.Module): + def __init__( + self, + in_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + spatial_compression: int, + **ignore_kwargs, + ): + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # Patcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.patcher = Patcher(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + in_channels = in_channels * patch_size * patch_size + + # calculate the number of downsample operations + self.num_downsamples = int(math.log2(spatial_compression)) - int(math.log2(patch_size)) + assert self.num_downsamples <= self.num_resolutions, ( + f"we can only downsample {self.num_resolutions} times at most" + ) + + # downsampling + self.conv_in = torch.nn.Conv2d(in_channels, channels, kernel_size=3, stride=1, padding=1) + + curr_res = resolution // patch_size + in_ch_mult = (1,) + tuple(channels_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = channels * in_ch_mult[i_level] + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + dropout=dropout, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level < self.num_downsamples: + down.downsample = Downsample(block_in) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in, dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in, dropout=dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d(block_in, z_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.patcher(x) + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1]) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level < self.num_downsamples: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + def __init__( + self, + out_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: int, + dropout: float, + resolution: int, + z_channels: int, + spatial_compression: int, + **ignore_kwargs, + ): + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # UnPatcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.unpatcher = UnPatcher(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + out_ch = out_channels * patch_size * patch_size + + # calculate the number of upsample operations + self.num_upsamples = int(math.log2(spatial_compression)) - int(math.log2(patch_size)) + assert self.num_upsamples <= self.num_resolutions, f"we can only upsample {self.num_resolutions} times at most" + + block_in = channels * channels_mult[self.num_resolutions - 1] + curr_res = (resolution // patch_size) // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + logging.info("Working with z of shape {} = {} dimensions.".format(self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = torch.nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in, dropout=dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in, dropout=dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + dropout=dropout, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(AttnBlock(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level >= (self.num_resolutions - self.num_upsamples): + up.upsample = Upsample(block_in) + curr_res = curr_res * 2 + self.up.insert(0, up) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level >= (self.num_resolutions - self.num_upsamples): + h = self.up[i_level].upsample(h) + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + h = self.unpatcher(h) + return h diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers2d_test.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers2d_test.py new file mode 100644 index 00000000..ac809ce8 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers2d_test.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The test for model definition of 2D layers + +PYTHONPATH=$PWD pytest -v cosmos1/models/tokenizer/modules/layers2d_test.py +""" + +import os + +import numpy as np +import pytest +import torch +from cosmos1.models.tokenizer.inference.image_lib import ImageTokenizer +from cosmos1.models.tokenizer.inference.utils import read_image +from cosmos1.models.tokenizer.networks import TokenizerConfigs +from torchvision.transforms import CenterCrop + + +# test configs +TEST_CONFIGS = [ + ("CI8x8", "nvidia/Cosmos-0.1-Tokenizer-CI8x8"), + ("CI16x16", "nvidia/Cosmos-0.1-Tokenizer-CI16x16"), + ("DI8x8", "nvidia/Cosmos-0.1-Tokenizer-DI8x8"), + ("DI16x16", "nvidia/Cosmos-0.1-Tokenizer-DI16x16"), + ("CI8x8-LowRes", "nvidia/Cosmos-1.0-Tokenizer-CI8x8-LowRes"), + ("CI16x16-LowRes", "nvidia/Cosmos-1.0-Tokenizer-CI16x16-LowRes"), + ("DI8x8-LowRes", "nvidia/Cosmos-1.0-Tokenizer-DI8x8-LowRes"), + ("DI16x16-LowRes", "nvidia/Cosmos-1.0-Tokenizer-DI16x16-LowRes"), +] + + +@pytest.fixture(scope="module") +def image_tensor(): + image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "test_data", "image.png") + print(f"image_path: {image_path}") + image = read_image(image_path) + + assert image.shape[0] >= 512, "Image height should be at least 512 pixels" + assert image.shape[1] >= 512, "Image width should be at least 512 pixels" + assert image.shape[2] == 3, "Image should have 3 channels" + + input_tensor = CenterCrop(512)( + torch.from_numpy(image[np.newaxis, ...]).to("cuda").to(torch.bfloat16).permute(0, 3, 1, 2) / 255.0 * 2.0 - 1.0 + ) + return input_tensor + + +@pytest.mark.parametrize("config", TEST_CONFIGS) +def test_tokenizer(config, image_tensor): + name, model_id = config + continuous = name.startswith(("C", "c")) + [ + spatial_compression, + ] = list(map(int, name[2:].split("x")[:1])) + print(f"\nTesting tokenizer: {model_id}") + print(f"spatial_compression={spatial_compression}") + print(f"checkpoint_enc=checkpoints/{os.path.basename(model_id)}/encoder.jit") + print(f"checkpoint_dec=checkpoints/{os.path.basename(model_id)}/decoder.jit") + + _config = TokenizerConfigs[name.replace("-", "_")].value + autoencoder = ImageTokenizer( + checkpoint_enc=f"checkpoints/{os.path.basename(model_id)}/encoder.jit", + checkpoint_dec=f"checkpoints/{os.path.basename(model_id)}/decoder.jit", + tokenizer_config=_config, + device="cuda", + dtype="bfloat16", + ) + + try: + # Test shape check + reconstructed_tensor = auto_shape_check(image_tensor, autoencoder, spatial_compression, continuous) + finally: + # Cleanup + del autoencoder + del reconstructed_tensor + torch.cuda.empty_cache() + torch.cuda.synchronize() + + +def auto_shape_check(input_tensor, autoencoder, spatial_compression, continuous): + if continuous: + (latent,) = autoencoder.encode(input_tensor) + torch.testing.assert_close(latent.shape, (1, 16, 512 // spatial_compression, 512 // spatial_compression)) + reconstructed_tensor = autoencoder.decode(latent) + else: + (indices, codes) = autoencoder.encode(input_tensor) + torch.testing.assert_close(indices.shape, (1, 512 // spatial_compression, 512 // spatial_compression)) + torch.testing.assert_close(codes.shape, (1, 6, 512 // spatial_compression, 512 // spatial_compression)) + reconstructed_tensor = autoencoder.decode(indices) + + torch.testing.assert_close(reconstructed_tensor.shape, input_tensor.shape) + return reconstructed_tensor diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers3d.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers3d.py new file mode 100644 index 00000000..d76d8c11 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers3d.py @@ -0,0 +1,955 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The model definition for 3D layers + +Adapted from: https://github.com/lucidrains/magvit2-pytorch/blob/ +9f49074179c912736e617d61b32be367eb5f993a/magvit2_pytorch/magvit2_pytorch.py#L889 + +[MIT License Copyright (c) 2023 Phil Wang] +https://github.com/lucidrains/magvit2-pytorch/blob/ +9f49074179c912736e617d61b32be367eb5f993a/LICENSE +""" + +import math +from typing import Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from cosmos1.models.tokenizer.modules.patching import Patcher, Patcher3D, UnPatcher, UnPatcher3D +from cosmos1.models.tokenizer.modules.utils import ( + CausalNormalize, + batch2space, + batch2time, + cast_tuple, + is_odd, + nonlinearity, + replication_pad, + space2batch, + time2batch, +) +from loguru import logger as logging + + +_LEGACY_NUM_GROUPS = 32 + + +class CausalConv3d(nn.Module): + def __init__( + self, + chan_in: int = 1, + chan_out: int = 1, + kernel_size: Union[int, Tuple[int, int, int]] = 3, + pad_mode: str = "constant", + **kwargs, + ): + super().__init__() + kernel_size = cast_tuple(kernel_size, 3) + + time_kernel_size, height_kernel_size, width_kernel_size = kernel_size + + assert is_odd(height_kernel_size) and is_odd(width_kernel_size) + + dilation = kwargs.pop("dilation", 1) + stride = kwargs.pop("stride", 1) + time_stride = kwargs.pop("time_stride", 1) + time_dilation = kwargs.pop("time_dilation", 1) + padding = kwargs.pop("padding", 1) + + self.pad_mode = pad_mode + time_pad = time_dilation * (time_kernel_size - 1) + (1 - time_stride) + self.time_pad = time_pad + + self.spatial_pad = (padding, padding, padding, padding) + + stride = (time_stride, stride, stride) + dilation = (time_dilation, dilation, dilation) + self.conv3d = nn.Conv3d( + chan_in, + chan_out, + kernel_size, + stride=stride, + dilation=dilation, + **kwargs, + ) + + def _replication_pad(self, x: torch.Tensor) -> torch.Tensor: + x_prev = x[:, :, :1, ...].repeat(1, 1, self.time_pad, 1, 1) + x = torch.cat([x_prev, x], dim=2) + padding = self.spatial_pad + (0, 0) + return F.pad(x, padding, mode=self.pad_mode, value=0.0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self._replication_pad(x) + return self.conv3d(x) + + +class CausalUpsample3d(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.conv = CausalConv3d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.repeat_interleave(2, dim=3).repeat_interleave(2, dim=4) + time_factor = 1.0 + 1.0 * (x.shape[2] > 1) + if isinstance(time_factor, torch.Tensor): + time_factor = time_factor.item() + x = x.repeat_interleave(int(time_factor), dim=2) + # TODO(freda): Check if this causes temporal inconsistency. + # Shoule reverse the order of the following two ops, + # better perf and better temporal smoothness. + x = self.conv(x) + return x[..., int(time_factor - 1) :, :, :] + + +class CausalDownsample3d(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.conv = CausalConv3d( + in_channels, + in_channels, + kernel_size=3, + stride=2, + time_stride=2, + padding=0, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad = (0, 1, 0, 1, 0, 0) + x = F.pad(x, pad, mode="constant", value=0) + x = replication_pad(x) + x = self.conv(x) + return x + + +class CausalHybridUpsample3d(nn.Module): + def __init__( + self, + in_channels: int, + spatial_up: bool = True, + temporal_up: bool = True, + **kwargs, + ) -> None: + super().__init__() + self.conv1 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(3, 1, 1), stride=1, time_stride=1, padding=0) + if temporal_up + else nn.Identity() + ) + self.conv2 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(1, 3, 3), stride=1, time_stride=1, padding=1) + if spatial_up + else nn.Identity() + ) + self.conv3 = ( + CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, time_stride=1, padding=0) + if spatial_up or temporal_up + else nn.Identity() + ) + self.spatial_up = spatial_up + self.temporal_up = temporal_up + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if not self.spatial_up and not self.temporal_up: + return x + + # hybrid upsample temporally. + if self.temporal_up: + time_factor = 1.0 + 1.0 * (x.shape[2] > 1) + if isinstance(time_factor, torch.Tensor): + time_factor = time_factor.item() + x = x.repeat_interleave(int(time_factor), dim=2) + x = x[..., int(time_factor - 1) :, :, :] + x = self.conv1(x) + x + + # hybrid upsample spatially. + if self.spatial_up: + x = x.repeat_interleave(2, dim=3).repeat_interleave(2, dim=4) + x = self.conv2(x) + x + + # final 1x1x1 conv. + x = self.conv3(x) + return x + + +class CausalHybridDownsample3d(nn.Module): + def __init__( + self, + in_channels: int, + spatial_down: bool = True, + temporal_down: bool = True, + **kwargs, + ) -> None: + super().__init__() + self.conv1 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(1, 3, 3), stride=2, time_stride=1, padding=0) + if spatial_down + else nn.Identity() + ) + self.conv2 = ( + CausalConv3d(in_channels, in_channels, kernel_size=(3, 1, 1), stride=1, time_stride=2, padding=0) + if temporal_down + else nn.Identity() + ) + self.conv3 = ( + CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, time_stride=1, padding=0) + if spatial_down or temporal_down + else nn.Identity() + ) + + self.spatial_down = spatial_down + self.temporal_down = temporal_down + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if not self.spatial_down and not self.temporal_down: + return x + + # hybrid downsample spatially. + if self.spatial_down: + pad = (0, 1, 0, 1, 0, 0) + x = F.pad(x, pad, mode="constant", value=0) + x1 = self.conv1(x) + x2 = F.avg_pool3d(x, kernel_size=(1, 2, 2), stride=(1, 2, 2)) + x = x1 + x2 + + # hybrid downsample temporally. + if self.temporal_down: + x = replication_pad(x) + x1 = self.conv2(x) + x2 = F.avg_pool3d(x, kernel_size=(2, 1, 1), stride=(2, 1, 1)) + x = x1 + x2 + + # final 1x1x1 conv. + x = self.conv3(x) + return x + + +class CausalResnetBlock3d(nn.Module): + def __init__( + self, + *, + in_channels: int, + out_channels: int = None, + dropout: float, + num_groups: int, + ) -> None: + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + + self.norm1 = CausalNormalize(in_channels, num_groups=num_groups) + self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = CausalNormalize(out_channels, num_groups=num_groups) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.nin_shortcut = ( + CausalConv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + if in_channels != out_channels + else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + x = self.nin_shortcut(x) + + return x + h + + +class CausalResnetBlockFactorized3d(nn.Module): + def __init__( + self, + *, + in_channels: int, + out_channels: int = None, + dropout: float, + num_groups: int, + ) -> None: + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + + self.norm1 = CausalNormalize(in_channels, num_groups=1) + self.conv1 = nn.Sequential( + CausalConv3d( + in_channels, + out_channels, + kernel_size=(1, 3, 3), + stride=1, + padding=1, + ), + CausalConv3d( + out_channels, + out_channels, + kernel_size=(3, 1, 1), + stride=1, + padding=0, + ), + ) + self.norm2 = CausalNormalize(out_channels, num_groups=num_groups) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = nn.Sequential( + CausalConv3d( + out_channels, + out_channels, + kernel_size=(1, 3, 3), + stride=1, + padding=1, + ), + CausalConv3d( + out_channels, + out_channels, + kernel_size=(3, 1, 1), + stride=1, + padding=0, + ), + ) + self.nin_shortcut = ( + CausalConv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + if in_channels != out_channels + else nn.Identity() + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + x = self.nin_shortcut(x) + + return x + h + + +class CausalAttnBlock(nn.Module): + def __init__(self, in_channels: int, num_groups: int) -> None: + super().__init__() + + self.norm = CausalNormalize(in_channels, num_groups=num_groups) + self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + q, batch_size = time2batch(q) + k, batch_size = time2batch(k) + v, batch_size = time2batch(v) + + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) + k = k.reshape(b, c, h * w) + w_ = torch.bmm(q, k) + w_ = w_ * (int(c) ** (-0.5)) + w_ = F.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) + h_ = torch.bmm(v, w_) + h_ = h_.reshape(b, c, h, w) + + h_ = batch2time(h_, batch_size) + h_ = self.proj_out(h_) + return x + h_ + + +class CausalTemporalAttnBlock(nn.Module): + def __init__(self, in_channels: int, num_groups: int) -> None: + super().__init__() + + self.norm = CausalNormalize(in_channels, num_groups=num_groups) + self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + q, batch_size, height = space2batch(q) + k, _, _ = space2batch(k) + v, _, _ = space2batch(v) + + bhw, c, t = q.shape + q = q.permute(0, 2, 1) # (bhw, t, c) + k = k.permute(0, 2, 1) # (bhw, t, c) + v = v.permute(0, 2, 1) # (bhw, t, c) + + w_ = torch.bmm(q, k.permute(0, 2, 1)) # (bhw, t, t) + w_ = w_ * (int(c) ** (-0.5)) + + # Apply causal mask + mask = torch.tril(torch.ones_like(w_)) + w_ = w_.masked_fill(mask == 0, float("-inf")) + w_ = F.softmax(w_, dim=2) + + # attend to values + h_ = torch.bmm(w_, v) # (bhw, t, c) + h_ = h_.permute(0, 2, 1).reshape(bhw, c, t) # (bhw, c, t) + + h_ = batch2space(h_, batch_size, height) + h_ = self.proj_out(h_) + return x + h_ + + +class EncoderBase(nn.Module): + def __init__( + self, + in_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + **ignore_kwargs, + ) -> None: + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # Patcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.patcher = Patcher(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + in_channels = in_channels * patch_size * patch_size + + # downsampling + self.conv_in = CausalConv3d(in_channels, channels, kernel_size=3, stride=1, padding=1) + + # num of groups for GroupNorm, num_groups=1 for LayerNorm. + num_groups = ignore_kwargs.get("num_groups", _LEGACY_NUM_GROUPS) + curr_res = resolution // patch_size + in_ch_mult = (1,) + tuple(channels_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = channels * in_ch_mult[i_level] + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks): + block.append( + CausalResnetBlock3d( + in_channels=block_in, + out_channels=block_out, + dropout=dropout, + num_groups=num_groups, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(CausalAttnBlock(block_in, num_groups=num_groups)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = CausalDownsample3d(block_in) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = CausalResnetBlock3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=num_groups, + ) + self.mid.attn_1 = CausalAttnBlock(block_in, num_groups=num_groups) + self.mid.block_2 = CausalResnetBlock3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=num_groups, + ) + + # end + self.norm_out = CausalNormalize(block_in, num_groups=num_groups) + self.conv_out = CausalConv3d(block_in, z_channels, kernel_size=3, stride=1, padding=1) + + def patcher3d(self, x: torch.Tensor) -> torch.Tensor: + x, batch_size = time2batch(x) + x = self.patcher(x) + x = batch2time(x, batch_size) + return x + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.patcher3d(x) + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1]) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + else: + # temporal downsample (last level) + time_factor = 1 + 1 * (hs[-1].shape[2] > 1) + if isinstance(time_factor, torch.Tensor): + time_factor = time_factor.item() + hs[-1] = replication_pad(hs[-1]) + hs.append( + F.avg_pool3d( + hs[-1], + kernel_size=[time_factor, 1, 1], + stride=[2, 1, 1], + ) + ) + + # middle + h = hs[-1] + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class DecoderBase(nn.Module): + def __init__( + self, + out_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + **ignore_kwargs, + ): + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # UnPatcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.unpatcher = UnPatcher(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + out_ch = out_channels * patch_size * patch_size + + block_in = channels * channels_mult[self.num_resolutions - 1] + curr_res = (resolution // patch_size) // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + logging.info("Working with z of shape {} = {} dimensions.".format(self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = CausalConv3d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + + # num of groups for GroupNorm, num_groups=1 for LayerNorm. + num_groups = ignore_kwargs.get("num_groups", _LEGACY_NUM_GROUPS) + + # middle + self.mid = nn.Module() + self.mid.block_1 = CausalResnetBlock3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=num_groups, + ) + self.mid.attn_1 = CausalAttnBlock(block_in, num_groups=num_groups) + self.mid.block_2 = CausalResnetBlock3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=num_groups, + ) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append( + CausalResnetBlock3d( + in_channels=block_in, + out_channels=block_out, + dropout=dropout, + num_groups=num_groups, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(CausalAttnBlock(block_in, num_groups=num_groups)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = CausalUpsample3d(block_in) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = CausalNormalize(block_in, num_groups=num_groups) + self.conv_out = CausalConv3d(block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def unpatcher3d(self, x: torch.Tensor) -> torch.Tensor: + x, batch_size = time2batch(x) + x = self.unpatcher(x) + x = batch2time(x, batch_size) + + return x + + def forward(self, z): + h = self.conv_in(z) + + # middle block. + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # decoder blocks. + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + else: + # temporal upsample (last level) + time_factor = 1.0 + 1.0 * (h.shape[2] > 1) + if isinstance(time_factor, torch.Tensor): + time_factor = time_factor.item() + h = h.repeat_interleave(int(time_factor), dim=2) + h = h[..., int(time_factor - 1) :, :, :] + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + h = self.unpatcher3d(h) + return h + + +class EncoderFactorized(nn.Module): + def __init__( + self, + in_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + spatial_compression: int = 16, + temporal_compression: int = 8, + **ignore_kwargs, + ) -> None: + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # Patcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.patcher3d = Patcher3D(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + in_channels = in_channels * patch_size * patch_size * patch_size + + # calculate the number of downsample operations + self.num_spatial_downs = int(math.log2(spatial_compression)) - int(math.log2(patch_size)) + assert self.num_spatial_downs <= self.num_resolutions, ( + f"Spatially downsample {self.num_resolutions} times at most" + ) + + self.num_temporal_downs = int(math.log2(temporal_compression)) - int(math.log2(patch_size)) + assert self.num_temporal_downs <= self.num_resolutions, ( + f"Temporally downsample {self.num_resolutions} times at most" + ) + + # downsampling + self.conv_in = nn.Sequential( + CausalConv3d( + in_channels, + channels, + kernel_size=(1, 3, 3), + stride=1, + padding=1, + ), + CausalConv3d(channels, channels, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + curr_res = resolution // patch_size + in_ch_mult = (1,) + tuple(channels_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = channels * in_ch_mult[i_level] + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks): + block.append( + CausalResnetBlockFactorized3d( + in_channels=block_in, + out_channels=block_out, + dropout=dropout, + num_groups=1, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append( + nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), + CausalTemporalAttnBlock(block_in, num_groups=1), + ) + ) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + spatial_down = i_level < self.num_spatial_downs + temporal_down = i_level < self.num_temporal_downs + down.downsample = CausalHybridDownsample3d( + block_in, + spatial_down=spatial_down, + temporal_down=temporal_down, + ) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = CausalResnetBlockFactorized3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=1, + ) + self.mid.attn_1 = nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), + CausalTemporalAttnBlock(block_in, num_groups=1), + ) + self.mid.block_2 = CausalResnetBlockFactorized3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=1, + ) + + # end + self.norm_out = CausalNormalize(block_in, num_groups=1) + self.conv_out = nn.Sequential( + CausalConv3d(block_in, z_channels, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d( + z_channels, + z_channels, + kernel_size=(3, 1, 1), + stride=1, + padding=0, + ), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.patcher3d(x) + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1]) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class DecoderFactorized(nn.Module): + def __init__( + self, + out_channels: int, + channels: int, + channels_mult: list[int], + num_res_blocks: int, + attn_resolutions: list[int], + dropout: float, + resolution: int, + z_channels: int, + spatial_compression: int = 16, + temporal_compression: int = 8, + **ignore_kwargs, + ): + super().__init__() + self.num_resolutions = len(channels_mult) + self.num_res_blocks = num_res_blocks + + # UnPatcher. + patch_size = ignore_kwargs.get("patch_size", 1) + self.unpatcher3d = UnPatcher3D(patch_size, ignore_kwargs.get("patch_method", "rearrange")) + out_ch = out_channels * patch_size * patch_size * patch_size + + # calculate the number of upsample operations + self.num_spatial_ups = int(math.log2(spatial_compression)) - int(math.log2(patch_size)) + assert self.num_spatial_ups <= self.num_resolutions, f"Spatially upsample {self.num_resolutions} times at most" + self.num_temporal_ups = int(math.log2(temporal_compression)) - int(math.log2(patch_size)) + assert self.num_temporal_ups <= self.num_resolutions, ( + f"Temporally upsample {self.num_resolutions} times at most" + ) + + block_in = channels * channels_mult[self.num_resolutions - 1] + curr_res = (resolution // patch_size) // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + logging.info("Working with z of shape {} = {} dimensions.".format(self.z_shape, np.prod(self.z_shape))) + + # z to block_in + self.conv_in = nn.Sequential( + CausalConv3d(z_channels, block_in, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(block_in, block_in, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + # middle + self.mid = nn.Module() + self.mid.block_1 = CausalResnetBlockFactorized3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=1, + ) + self.mid.attn_1 = nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), + CausalTemporalAttnBlock(block_in, num_groups=1), + ) + self.mid.block_2 = CausalResnetBlockFactorized3d( + in_channels=block_in, + out_channels=block_in, + dropout=dropout, + num_groups=1, + ) + + legacy_mode = ignore_kwargs.get("legacy_mode", False) + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = channels * channels_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append( + CausalResnetBlockFactorized3d( + in_channels=block_in, + out_channels=block_out, + dropout=dropout, + num_groups=1, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append( + nn.Sequential( + CausalAttnBlock(block_in, num_groups=1), + CausalTemporalAttnBlock(block_in, num_groups=1), + ) + ) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + # The layer index for temporal/spatial downsampling performed + # in the encoder should correspond to the layer index in + # reverse order where upsampling is performed in the decoder. + # If you've a pre-trained model, you can simply finetune. + i_level_reverse = self.num_resolutions - i_level - 1 + if legacy_mode: + temporal_up = i_level_reverse < self.num_temporal_ups + else: + temporal_up = 0 < i_level_reverse < self.num_temporal_ups + 1 + spatial_up = temporal_up or ( + i_level_reverse < self.num_spatial_ups and self.num_spatial_ups > self.num_temporal_ups + ) + up.upsample = CausalHybridUpsample3d(block_in, spatial_up=spatial_up, temporal_up=temporal_up) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = CausalNormalize(block_in, num_groups=1) + self.conv_out = nn.Sequential( + CausalConv3d(block_in, out_ch, kernel_size=(1, 3, 3), stride=1, padding=1), + CausalConv3d(out_ch, out_ch, kernel_size=(3, 1, 1), stride=1, padding=0), + ) + + def forward(self, z): + h = self.conv_in(z) + + # middle block. + h = self.mid.block_1(h) + h = self.mid.attn_1(h) + h = self.mid.block_2(h) + + # decoder blocks. + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + h = self.unpatcher3d(h) + return h diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers3d_test.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers3d_test.py new file mode 100644 index 00000000..ac038a6c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/layers3d_test.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The test for model definition of 3D layers + +PYTHONPATH=$PWD pytest -v cosmos1/models/tokenizer/modules/layers3d_test.py +""" + +import os + +import numpy as np +import pytest +import torch +from cosmos1.models.tokenizer.inference.utils import read_video +from cosmos1.models.tokenizer.inference.video_lib import CausalVideoTokenizer +from cosmos1.models.tokenizer.networks import TokenizerConfigs +from torchvision.transforms import CenterCrop + + +# test configs +TEST_CONFIGS = [ + ("CV4x8x8", "nvidia/Cosmos-0.1-Tokenizer-CV4x8x8"), + ("CV8x8x8", "nvidia/Cosmos-0.1-Tokenizer-CV8x8x8"), + ("CV8x16x16", "nvidia/Cosmos-0.1-Tokenizer-CV8x16x16"), + ("DV4x8x8", "nvidia/Cosmos-0.1-Tokenizer-DV4x8x8"), + ("DV8x8x8", "nvidia/Cosmos-0.1-Tokenizer-DV8x8x8"), + ("DV8x16x16", "nvidia/Cosmos-0.1-Tokenizer-DV8x16x16"), + ("CV8x8x8", "nvidia/Cosmos-1.0-Tokenizer-CV8x8x8"), + ("DV8x16x16", "nvidia/Cosmos-1.0-Tokenizer-DV8x16x16"), + ("CV4x8x8-LowRes", "nvidia/Cosmos-1.0-Tokenizer-CV4x8x8-LowRes"), + ("DV4x8x8-LowRes", "nvidia/Cosmos-1.0-Tokenizer-DV4x8x8-LowRes"), +] + + +@pytest.fixture(scope="module") +def video_tensor(): + video_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "test_data", "video.mp4") + print(f"video_path: {video_path}") + video = read_video(video_path) + + assert video.shape[0] >= 17, "Video length should be at least 17 frames" + assert video.shape[1] >= 512, "Video height should be at least 512 pixels" + assert video.shape[2] >= 512, "Video width should be at least 512 pixels" + assert video.shape[3] == 3, "Video should have 3 channels" + + input_tensor = CenterCrop(512)( + torch.from_numpy(video[np.newaxis, ...])[:, :17].to("cuda").to(torch.bfloat16).permute(0, 4, 1, 2, 3) + / 255.0 + * 2.0 + - 1.0 + ) + return input_tensor + + +@pytest.mark.parametrize("config", TEST_CONFIGS) +def test_tokenizer(config, video_tensor): + name, model_id = config + continuous = name.startswith(("C", "c")) + temporal_compression, spatial_compression = list(map(int, name[2:].split("x")[:2])) + print(f"\nTesting tokenizer: {model_id}") + print(f"temporal_compression={temporal_compression}") + print(f"spatial_compression={spatial_compression}") + print(f"checkpoint_enc=checkpoints/{os.path.basename(model_id)}/encoder.jit") + print(f"checkpoint_dec=checkpoints/{os.path.basename(model_id)}/decoder.jit") + + _config = TokenizerConfigs[name.replace("-", "_")].value + autoencoder = CausalVideoTokenizer( + checkpoint_enc=f"checkpoints/{os.path.basename(model_id)}/encoder.jit", + checkpoint_dec=f"checkpoints/{os.path.basename(model_id)}/decoder.jit", + tokenizer_config=_config, + device="cuda", + dtype="bfloat16", + ) + + try: + # Test shape check + reconstructed_tensor = auto_shape_check( + video_tensor, autoencoder, temporal_compression, spatial_compression, continuous + ) + finally: + # Cleanup + del autoencoder + del reconstructed_tensor + torch.cuda.empty_cache() + torch.cuda.synchronize() + + +def auto_shape_check(input_tensor, autoencoder, temporal_compression, spatial_compression, continuous): + if continuous: + (latent,) = autoencoder.encode(input_tensor) + torch.testing.assert_close( + latent.shape, + (1, 16, (17 - 1) // temporal_compression + 1, 512 // spatial_compression, 512 // spatial_compression), + ) + reconstructed_tensor = autoencoder.decode(latent) + else: + (indices, codes) = autoencoder.encode(input_tensor) + torch.testing.assert_close( + indices.shape, + (1, (17 - 1) // temporal_compression + 1, 512 // spatial_compression, 512 // spatial_compression), + ) + torch.testing.assert_close( + codes.shape, + (1, 6, (17 - 1) // temporal_compression + 1, 512 // spatial_compression, 512 // spatial_compression), + ) + reconstructed_tensor = autoencoder.decode(indices) + + torch.testing.assert_close(reconstructed_tensor.shape, input_tensor.shape) + return reconstructed_tensor diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/patching.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/patching.py new file mode 100644 index 00000000..53395562 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/patching.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The patcher and unpatcher implementation for 2D and 3D data. + +The idea of Haar wavelet is to compute LL, LH, HL, HH component as two 1D convolutions. +One on the rows and one on the columns. +For example, in 1D signal, we have [a, b], then the low-freq compoenent is [a + b] / 2 and high-freq is [a - b] / 2. +We can use a 1D convolution with kernel [1, 1] and stride 2 to represent the L component. +For H component, we can use a 1D convolution with kernel [1, -1] and stride 2. +Although in principle, we typically only do additional Haar wavelet over the LL component. But here we do it for all + as we need to support downsampling for more than 2x. +For example, 4x downsampling can be done by 2x Haar and additional 2x Haar, and the shape would be. + [3, 256, 256] -> [12, 128, 128] -> [48, 64, 64] +""" + +import torch +import torch.nn.functional as F +from einops import rearrange + + +_WAVELETS = { + "haar": torch.tensor([0.7071067811865476, 0.7071067811865476]), + "rearrange": torch.tensor([1.0, 1.0]), +} +_PERSISTENT = True + + +class Patcher(torch.nn.Module): + """A module to convert image tensors into patches using torch operations. + + The main difference from `class Patching` is that this module implements + all operations using torch, rather than python or numpy, for efficiency purpose. + + It's bit-wise identical to the Patching module outputs, with the added + benefit of being torch.jit scriptable. + """ + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__() + self.patch_size = patch_size + self.patch_method = patch_method + self.register_buffer("wavelets", _WAVELETS[patch_method], persistent=_PERSISTENT) + self.range = range(int(torch.log2(torch.tensor(self.patch_size)).item())) + self.register_buffer( + "_arange", + torch.arange(_WAVELETS[patch_method].shape[0]), + persistent=_PERSISTENT, + ) + for param in self.parameters(): + param.requires_grad = False + + def forward(self, x): + if self.patch_method == "haar": + return self._haar(x) + elif self.patch_method == "rearrange": + return self._arrange(x) + else: + raise ValueError("Unknown patch method: " + self.patch_method) + + def _dwt(self, x, mode="reflect", rescale=False): + dtype = x.dtype + h = self.wavelets + + n = h.shape[0] + g = x.shape[1] + hl = h.flip(0).reshape(1, 1, -1).repeat(g, 1, 1) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hh = hh.to(dtype=dtype) + hl = hl.to(dtype=dtype) + + x = F.pad(x, pad=(n - 2, n - 1, n - 2, n - 1), mode=mode).to(dtype) + xl = F.conv2d(x, hl.unsqueeze(2), groups=g, stride=(1, 2)) + xh = F.conv2d(x, hh.unsqueeze(2), groups=g, stride=(1, 2)) + xll = F.conv2d(xl, hl.unsqueeze(3), groups=g, stride=(2, 1)) + xlh = F.conv2d(xl, hh.unsqueeze(3), groups=g, stride=(2, 1)) + xhl = F.conv2d(xh, hl.unsqueeze(3), groups=g, stride=(2, 1)) + xhh = F.conv2d(xh, hh.unsqueeze(3), groups=g, stride=(2, 1)) + + out = torch.cat([xll, xlh, xhl, xhh], dim=1) + if rescale: + out = out / 2 + return out + + def _haar(self, x): + for _ in self.range: + x = self._dwt(x, rescale=True) + return x + + def _arrange(self, x): + x = rearrange( + x, + "b c (h p1) (w p2) -> b (c p1 p2) h w", + p1=self.patch_size, + p2=self.patch_size, + ).contiguous() + return x + + +class Patcher3D(Patcher): + """A 3D discrete wavelet transform for video data, expects 5D tensor, i.e. a batch of videos.""" + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__(patch_method=patch_method, patch_size=patch_size) + self.register_buffer( + "patch_size_buffer", + patch_size * torch.ones([1], dtype=torch.int32), + persistent=_PERSISTENT, + ) + + def _dwt(self, x, wavelet, mode="reflect", rescale=False): + dtype = x.dtype + h = self.wavelets + + n = h.shape[0] + g = x.shape[1] + hl = h.flip(0).reshape(1, 1, -1).repeat(g, 1, 1) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hh = hh.to(dtype=dtype) + hl = hl.to(dtype=dtype) + + # Handles temporal axis. + x = F.pad(x, pad=(max(0, n - 2), n - 1, n - 2, n - 1, n - 2, n - 1), mode=mode).to(dtype) + xl = F.conv3d(x, hl.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + xh = F.conv3d(x, hh.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + + # Handles spatial axes. + xll = F.conv3d(xl, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xlh = F.conv3d(xl, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xhl = F.conv3d(xh, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xhh = F.conv3d(xh, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + + xlll = F.conv3d(xll, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xllh = F.conv3d(xll, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xlhl = F.conv3d(xlh, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xlhh = F.conv3d(xlh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhll = F.conv3d(xhl, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhlh = F.conv3d(xhl, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhhl = F.conv3d(xhh, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhhh = F.conv3d(xhh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + out = torch.cat([xlll, xllh, xlhl, xlhh, xhll, xhlh, xhhl, xhhh], dim=1) + if rescale: + out = out / (2 * torch.sqrt(torch.tensor(2.0))) + return out + + def _haar(self, x): + xi, xv = torch.split(x, [1, x.shape[2] - 1], dim=2) + x = torch.cat([xi.repeat_interleave(self.patch_size, dim=2), xv], dim=2) + for _ in self.range: + x = self._dwt(x, "haar", rescale=True) + return x + + def _arrange(self, x): + xi, xv = torch.split(x, [1, x.shape[2] - 1], dim=2) + x = torch.cat([xi.repeat_interleave(self.patch_size, dim=2), xv], dim=2) + x = rearrange( + x, + "b c (t p1) (h p2) (w p3) -> b (c p1 p2 p3) t h w", + p1=self.patch_size, + p2=self.patch_size, + p3=self.patch_size, + ).contiguous() + return x + + +class UnPatcher(torch.nn.Module): + """A module to convert patches into image tensorsusing torch operations. + + The main difference from `class Unpatching` is that this module implements + all operations using torch, rather than python or numpy, for efficiency purpose. + + It's bit-wise identical to the Unpatching module outputs, with the added + benefit of being torch.jit scriptable. + """ + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__() + self.patch_size = patch_size + self.patch_method = patch_method + self.register_buffer("wavelets", _WAVELETS[patch_method], persistent=_PERSISTENT) + self.range = range(int(torch.log2(torch.tensor(self.patch_size)).item())) + self.register_buffer( + "_arange", + torch.arange(_WAVELETS[patch_method].shape[0]), + persistent=_PERSISTENT, + ) + for param in self.parameters(): + param.requires_grad = False + + def forward(self, x): + if self.patch_method == "haar": + return self._ihaar(x) + elif self.patch_method == "rearrange": + return self._iarrange(x) + else: + raise ValueError("Unknown patch method: " + self.patch_method) + + def _idwt(self, x, wavelet="haar", mode="reflect", rescale=False): + dtype = x.dtype + h = self.wavelets + n = h.shape[0] + + g = x.shape[1] // 4 + hl = h.flip([0]).reshape(1, 1, -1).repeat([g, 1, 1]) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hh = hh.to(dtype=dtype) + hl = hl.to(dtype=dtype) + + xll, xlh, xhl, xhh = torch.chunk(x.to(dtype), 4, dim=1) + + # Inverse transform. + yl = torch.nn.functional.conv_transpose2d(xll, hl.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + yl += torch.nn.functional.conv_transpose2d(xlh, hh.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + yh = torch.nn.functional.conv_transpose2d(xhl, hl.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + yh += torch.nn.functional.conv_transpose2d(xhh, hh.unsqueeze(3), groups=g, stride=(2, 1), padding=(n - 2, 0)) + y = torch.nn.functional.conv_transpose2d(yl, hl.unsqueeze(2), groups=g, stride=(1, 2), padding=(0, n - 2)) + y += torch.nn.functional.conv_transpose2d(yh, hh.unsqueeze(2), groups=g, stride=(1, 2), padding=(0, n - 2)) + + if rescale: + y = y * 2 + return y + + def _ihaar(self, x): + for _ in self.range: + x = self._idwt(x, "haar", rescale=True) + return x + + def _iarrange(self, x): + x = rearrange( + x, + "b (c p1 p2) h w -> b c (h p1) (w p2)", + p1=self.patch_size, + p2=self.patch_size, + ) + return x + + +class UnPatcher3D(UnPatcher): + """A 3D inverse discrete wavelet transform for video wavelet decompositions.""" + + def __init__(self, patch_size=1, patch_method="haar"): + super().__init__(patch_method=patch_method, patch_size=patch_size) + + def _idwt(self, x, wavelet="haar", mode="reflect", rescale=False): + dtype = x.dtype + h = self.wavelets + + g = x.shape[1] // 8 # split into 8 spatio-temporal filtered tesnors. + hl = h.flip([0]).reshape(1, 1, -1).repeat([g, 1, 1]) + hh = (h * ((-1) ** self._arange)).reshape(1, 1, -1).repeat(g, 1, 1) + hl = hl.to(dtype=dtype) + hh = hh.to(dtype=dtype) + + xlll, xllh, xlhl, xlhh, xhll, xhlh, xhhl, xhhh = torch.chunk(x, 8, dim=1) + + # Height height transposed convolutions. + xll = F.conv_transpose3d(xlll, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xll += F.conv_transpose3d(xllh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + xlh = F.conv_transpose3d(xlhl, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xlh += F.conv_transpose3d(xlhh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + xhl = F.conv_transpose3d(xhll, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhl += F.conv_transpose3d(xhlh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + xhh = F.conv_transpose3d(xhhl, hl.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + xhh += F.conv_transpose3d(xhhh, hh.unsqueeze(2).unsqueeze(3), groups=g, stride=(1, 1, 2)) + + # Handles width transposed convolutions. + xl = F.conv_transpose3d(xll, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xl += F.conv_transpose3d(xlh, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xh = F.conv_transpose3d(xhl, hl.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + xh += F.conv_transpose3d(xhh, hh.unsqueeze(2).unsqueeze(4), groups=g, stride=(1, 2, 1)) + + # Handles time axis transposed convolutions. + x = F.conv_transpose3d(xl, hl.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + x += F.conv_transpose3d(xh, hh.unsqueeze(3).unsqueeze(4), groups=g, stride=(2, 1, 1)) + + if rescale: + x = x * (2 * torch.sqrt(torch.tensor(2.0))) + return x + + def _ihaar(self, x): + for _ in self.range: + x = self._idwt(x, "haar", rescale=True) + x = x[:, :, self.patch_size - 1 :, ...] + return x + + def _iarrange(self, x): + x = rearrange( + x, + "b (c p1 p2 p3) t h w -> b c (t p1) (h p2) (w p3)", + p1=self.patch_size, + p2=self.patch_size, + p3=self.patch_size, + ) + x = x[:, :, self.patch_size - 1 :, ...] + return x diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/quantizers.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/quantizers.py new file mode 100644 index 00000000..231f4c27 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/quantizers.py @@ -0,0 +1,510 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""Quantizers for discrete image and video tokenization.""" + +from typing import Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from cosmos1.models.tokenizer.modules.utils import default, entropy, pack_one, rearrange, round_ste, unpack_one +from einops import reduce +from loguru import logger as logging + + +_PERSISTENT = True + + +class ResidualFSQuantizer(nn.Module): + """Residual Finite Scalar Quantization + + Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf + """ + + def __init__(self, levels: list[int], num_quantizers: int, **ignore_kwargs): + super().__init__() + self.dtype = ignore_kwargs.get("dtype", torch.float32) + self.layers = nn.ModuleList([FSQuantizer(levels=levels) for _ in range(num_quantizers)]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + indices_stack = [] + residual = x + quantized_out = 0 + loss_out = 0 + for i, layer in enumerate(self.layers): + quant_indices, z, loss = layer(residual) + indices_stack.append(quant_indices) + residual = residual - z.detach() + quantized_out = quantized_out + z + loss_out = loss_out + loss + self.residual = residual + indices = torch.stack(indices_stack, dim=1) + return indices, quantized_out.to(self.dtype), loss_out.to(self.dtype) + + def indices_to_codes(self, indices_stack: torch.Tensor) -> torch.Tensor: + quantized_out = 0 + for layer, indices in zip(self.layers, indices_stack.transpose(0, 1)): + quantized_out += layer.indices_to_codes(indices) + return quantized_out + + +class FSQuantizer(nn.Module): + """Finite Scalar Quantization: VQ-VAE Made Simple - https://arxiv.org/abs/2309.15505 + + Code adapted from Jax version in Appendix A.1. + + Adapted from: https://github.com/lucidrains/vector-quantize-pytorch/blob/9502a1f447876d53fd37685b226bf28f250dc4a3/ + vector_quantize_pytorch/finite_scalar_quantization.py + [Copyright (c) 2020 Phil Wang] + https://github.com/lucidrains/vector-quantize-pytorch/blob/9502a1f447876d53fd37685b226bf28f250dc4a3/LICENSE + """ + + def __init__( + self, + levels: list[int], + dim: Optional[int] = None, + num_codebooks=1, + keep_num_codebooks_dim: Optional[bool] = None, + scale: Optional[float] = None, + **ignore_kwargs, + ): + super().__init__() + self.dtype = ignore_kwargs.get("dtype", torch.bfloat16) + _levels = torch.tensor(levels, dtype=torch.int32) + self.register_buffer("_levels", _levels, persistent=_PERSISTENT) + + _basis = torch.cumprod(torch.tensor([1] + levels[:-1]), dim=0, dtype=torch.int32) + self.register_buffer("_basis", _basis, persistent=_PERSISTENT) + + self.scale = scale + + codebook_dim = len(levels) + self.codebook_dim = codebook_dim + + effective_codebook_dim = codebook_dim * num_codebooks + self.num_codebooks = num_codebooks + self.effective_codebook_dim = effective_codebook_dim + + keep_num_codebooks_dim = default(keep_num_codebooks_dim, num_codebooks > 1) + assert not (num_codebooks > 1 and not keep_num_codebooks_dim) + self.keep_num_codebooks_dim = keep_num_codebooks_dim + + self.dim = default(dim, len(_levels) * num_codebooks) + + has_projections = self.dim != effective_codebook_dim + self.project_in = nn.Linear(self.dim, effective_codebook_dim) if has_projections else nn.Identity() + self.project_out = nn.Linear(effective_codebook_dim, self.dim) if has_projections else nn.Identity() + self.has_projections = has_projections + + self.codebook_size = self._levels.prod().item() + + implicit_codebook = self.indices_to_codes(torch.arange(self.codebook_size), project_out=False) + self.register_buffer("implicit_codebook", implicit_codebook, persistent=_PERSISTENT) + + def bound(self, z: torch.Tensor, eps: float = 1e-3) -> torch.Tensor: + """Bound `z`, an array of shape (..., d).""" + half_l = (self._levels - 1) * (1 + eps) / 2 + offset = torch.where(self._levels % 2 == 0, 0.5, 0.0) + shift = (offset / half_l).atanh() + return (z + shift).tanh() * half_l - offset + + def quantize(self, z: torch.Tensor) -> torch.Tensor: + """Quantizes z, returns quantized zhat, same shape as z.""" + quantized = round_ste(self.bound(z)) + half_width = self._levels // 2 # Renormalize to [-1, 1]. + return quantized / half_width + + def _scale_and_shift(self, zhat_normalized: torch.Tensor) -> torch.Tensor: + half_width = self._levels // 2 + return (zhat_normalized * half_width) + half_width + + def _scale_and_shift_inverse(self, zhat: torch.Tensor) -> torch.Tensor: + half_width = self._levels // 2 + return (zhat - half_width) / half_width + + def codes_to_indices(self, zhat: torch.Tensor) -> torch.Tensor: + """Converts a `code` to an index in the codebook.""" + assert zhat.shape[-1] == self.codebook_dim + zhat = self._scale_and_shift(zhat).float() + return (zhat * self._basis).sum(dim=-1).to(torch.int32) + + def indices_to_codes(self, indices: torch.Tensor, project_out=True) -> torch.Tensor: + """Inverse of `codes_to_indices`.""" + is_img_or_video = indices.ndim >= (3 + int(self.keep_num_codebooks_dim)) + indices = rearrange(indices, "... -> ... 1") + codes_non_centered = (indices // self._basis) % self._levels + codes = self._scale_and_shift_inverse(codes_non_centered) + + if self.keep_num_codebooks_dim: + codes = rearrange(codes, "... c d -> ... (c d)") + + if project_out: + codes = self.project_out(codes) + + if is_img_or_video: + codes = rearrange(codes, "b ... d -> b d ...") + + return codes.to(self.dtype) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + """ + einstein notation + b - batch + n - sequence (or flattened spatial dimensions) + d - feature dimension, which is also log2(codebook size) + c - number of codebook dim + """ + is_img_or_video = z.ndim >= 4 + + # standardize image or video into (batch, seq, dimension) + + if is_img_or_video: + z = rearrange(z, "b d ... -> b ... d") + z, ps = pack_one(z, "b * d") + + assert z.shape[-1] == self.dim, f"expected dimension of {self.dim} but found dimension of {z.shape[-1]}" + + z = self.project_in(z) + + z = rearrange(z, "b n (c d) -> b n c d", c=self.num_codebooks) + + codes = self.quantize(z) + indices = self.codes_to_indices(codes) + + codes = rearrange(codes, "b n c d -> b n (c d)") + + out = self.project_out(codes) + + # reconstitute image or video dimensions + + if is_img_or_video: + out = unpack_one(out, ps, "b * d") + out = rearrange(out, "b ... d -> b d ...") + indices = unpack_one(indices, ps, "b * c") + dummy_loss = torch.zeros_like(out.mean(dim=[1, 2, 3], keepdim=True)) + else: + dummy_loss = torch.zeros_like(out.mean(dim=[1, 2], keepdim=True)).unsqueeze(1) + + if not self.keep_num_codebooks_dim: + indices = rearrange(indices, "... 1 -> ...") + + return (indices, out.to(self.dtype), dummy_loss) + + +class VectorQuantizer(nn.Module): + """Improved version over VectorQuantizer. Mostly + avoids costly matrix multiplications and allows for post-hoc remapping of indices. + + Adapted from: https://github.com/CompVis/taming-transformers/blob/3ba01b241669f5ade541ce990f7650a3b8f65318/ + taming/modules/vqvae/quantize.py + + [Copyright (c) 2020 Patrick Esser and Robin Rombach and Björn Ommer] + https://github.com/CompVis/taming-transformers/blob/3ba01b241669f5ade541ce990f7650a3b8f65318/License.txt + """ + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + beta: float = 0.25, + remap: str = None, + unknown_index: str = "random", + sane_index_shape: bool = False, + legacy: bool = True, + use_norm=False, + **ignore_kwargs, + ): + super().__init__() + self.n_e = num_embeddings + self.e_dim = embedding_dim + self.beta = beta + self.legacy = legacy + self.norm = lambda x: F.normalize(x, dim=-1) if use_norm else x + + self.embedding = nn.Embedding(self.n_e, self.e_dim) + self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e) + + self.remap = remap + if self.remap is not None: + self.register_buffer("used", torch.tensor(np.load(self.remap))) + self.re_embed = self.used.shape[0] + self.unknown_index = unknown_index + if self.unknown_index == "extra": + self.unknown_index = self.re_embed + self.re_embed = self.re_embed + 1 + print( + f"Remapping {self.n_e} indices to {self.re_embed} indices. " + f"Using {self.unknown_index} for unknown indices." + ) + else: + self.re_embed = num_embeddings + + self.sane_index_shape = sane_index_shape + self.dtype = ignore_kwargs.get("dtype", torch.float32) + + def remap_to_used(self, inds): + ishape = inds.shape + assert len(ishape) > 1 + inds = inds.reshape(ishape[0], -1) + used = self.used.to(inds) + match = (inds[:, :, None] == used[None, None, ...]).long() + new = match.argmax(-1) + unknown = match.sum(2) < 1 + if self.unknown_index == "random": + new[unknown] = torch.randint(0, self.re_embed, size=new[unknown].shape).to(device=new.device) + else: + new[unknown] = self.unknown_index + return new.reshape(ishape) + + def unmap_to_all(self, inds): + ishape = inds.shape + assert len(ishape) > 1 + inds = inds.reshape(ishape[0], -1) + used = self.used.to(inds) + if self.re_embed > self.used.shape[0]: # extra token + inds[inds >= self.used.shape[0]] = 0 # simply set to zero + back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds) + return back.reshape(ishape) + + def forward(self, z, temp=None, rescale_logits=False, return_logits=False): + assert temp is None or temp == 1.0, "Only for interface compatible with Gumbel" + assert rescale_logits is False, "Only for interface compatible with Gumbel" + assert return_logits is False, "Only for interface compatible with Gumbel" + z = rearrange(z, "b c h w -> b h w c").contiguous() + z_flattened = z.view(-1, self.e_dim) + + d = ( + torch.sum(z_flattened**2, dim=1, keepdim=True) + + torch.sum(self.embedding.weight**2, dim=1) + - 2 + * torch.einsum( + "bd,dn->bn", + z_flattened, + rearrange(self.embedding.weight, "n d -> d n"), + ) + ) + + encoding_indices = torch.argmin(d, dim=1).unsqueeze(1) + encodings = torch.zeros(encoding_indices.shape[0], self.n_e, device=z.device) + encodings.scatter_(1, encoding_indices, 1) + z_q = torch.matmul(encodings, self.embedding.weight).view(z.shape) + min_encodings = None + + z_q, z = self.norm(z_q), self.norm(z) + + # compute loss for embedding + commit_loss = torch.mean((z_q - z.detach()) ** 2, dim=[1, 2, 3], keepdim=True) + emb_loss = torch.mean((z_q.detach() - z) ** 2, dim=[1, 2, 3], keepdim=True) + if not self.legacy: + loss = self.beta * emb_loss + commit_loss + else: + loss = emb_loss + self.beta * commit_loss + + # preserve gradients + z_q = z + (z_q - z).detach() + avg_probs = torch.mean(encodings, dim=0) + perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10))) + + # reshape back to match original input shape + z_q = rearrange(z_q, "b h w c -> b c h w").contiguous() + + if self.remap is not None: + min_encoding_indices = encoding_indices.squeeze(1).reshape(z.shape[0], -1) # add batch axis + min_encoding_indices = self.remap_to_used(encoding_indices.squeeze(1)) + min_encoding_indices = min_encoding_indices.reshape(-1, 1) # flatten + + if self.sane_index_shape: + min_encoding_indices = min_encoding_indices.reshape(z_q.shape[0], z_q.shape[2], z_q.shape[3]) + + # TODO: return (indices, z_q, loss) + return ( + z_q, + loss, + ( + encoding_indices.squeeze(1), + min_encodings, + commit_loss.mean().detach(), + self.beta * emb_loss.mean().detach(), + perplexity.mean().detach(), + ), + ) + + def get_codebook_entry(self, indices, shape): + # shape specifying (batch, height, width, channel) + if self.remap is not None: + indices = indices.reshape(shape[0], -1) # add batch axis + indices = self.unmap_to_all(indices) + indices = indices.reshape(-1) # flatten again + + # get quantized latent vectors + z_q = self.embedding(indices) + + if shape is not None: + z_q = z_q.view(shape) + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q + + +class LFQuantizer(nn.Module): + """Lookup-Free Quantization + + Adapted from: https://github.com/lucidrains/vector-quantize-pytorch/blob/9502a1f447876d53fd37685b226bf28f250dc4a3/ + vector_quantize_pytorch/lookup_free_quantization.py + [Copyright (c) 2020 Phil Wang] + https://github.com/lucidrains/vector-quantize-pytorch/blob/9502a1f447876d53fd37685b226bf28f250dc4a3/LICENSE + """ + + def __init__( + self, + *, + codebook_size: int, + codebook_dim: int, + embed_dim: Optional[int] = None, # if None, use codebook_dim + entropy_loss_weight=0.1, + commitment_loss_weight=0.25, + default_temp: float = 0.01, + entropy_loss: bool = False, + **ignore_kwargs, + ): + """Lookup-Free Quantization + + Args: + codebook_size (int): The number of entries in the codebook. + codebook_dim (int): The number of bits in each code. + embed_dim (Optional[int], optional): The dimension of the input embedding. Defaults to None. + entropy_loss_weight (float, optional): Whether to use entropy loss. Defaults to 0.1. + commitment_loss_weight (float, optional): Weight for commitment loss. Defaults to 0.25. + default_temp (float, optional): The temprature to use. Defaults to 0.01. + entropy_loss (bool, optional): Flag for entropy loss. Defaults to False. + """ + super().__init__() + self.entropy_loss = entropy_loss + self.codebook_dim = codebook_dim + self.default_temp = default_temp + self.entrop_loss_weight = entropy_loss_weight + self.commitment_loss_weight = commitment_loss_weight + embed_dim = embed_dim or codebook_dim + + has_projections = embed_dim != codebook_dim + self.project_in = nn.Linear(embed_dim, codebook_dim) if has_projections else nn.Identity() + self.project_out = nn.Linear(codebook_dim, embed_dim) if has_projections else nn.Identity() + logging.info(f"LFQ: has_projections={has_projections}, dim_in={embed_dim}, codebook_dim={codebook_dim}") + + self.dtype = ignore_kwargs.get("dtype", torch.float32) + + if entropy_loss: + assert 2**codebook_dim == codebook_size, "codebook size must be 2 ** codebook_dim" + self.codebook_size = codebook_size + + self.register_buffer( + "mask", + 2 ** torch.arange(codebook_dim - 1, -1, -1), + persistent=_PERSISTENT, + ) + self.register_buffer("zero", torch.tensor(0.0), persistent=_PERSISTENT) + + all_codes = torch.arange(codebook_size) + bits = ((all_codes[..., None].int() & self.mask) != 0).float() + codebook = 2 * bits - 1.0 + + self.register_buffer("codebook", codebook, persistent=_PERSISTENT) # [codebook_size, codebook_dim] + + def forward(self, z: torch.Tensor, temp: float = None) -> torch.Tensor: + temp = temp or self.default_temp + + z = rearrange(z, "b d ... -> b ... d") + z, ps = pack_one(z, "b * d") + z = self.project_in(z) + + # split out number of codebooks + z = rearrange(z, "b n (c d) -> b n c d", c=self.num_codebooks) + + # quantization + original_input = z + + codebook_value = torch.ones_like(z) + z_q = torch.where(z > 0, codebook_value, -codebook_value) + + # preserve gradients + z_q = z + (z_q - z).detach() + + # commit loss + commit_loss = ((original_input - z_q.detach()) ** 2).mean(dim=[1, 2, 3]) + + z_q = rearrange(z_q, "b n c d -> b n (c d)") + z_q = self.project_out(z_q) + + # reshape + z_q = unpack_one(z_q, ps, "b * d") + z_q = rearrange(z_q, "b ... d -> b d ...") + + loss = self.commitment_loss_weight * commit_loss + + # entropy loss (eq-5) + if self.entropy_loss: + # indices + indices = reduce((z > 0).int() * self.mask.int(), "b n c d -> b n c", "sum") + indices = unpack_one(indices, ps, "b * c") + indices = rearrange(indices, "... 1 -> ...") + + distance = -2 * torch.einsum( + "... i d, j d -> ... i j", + original_input, + self.codebook.to(original_input.dtype), + ) + prob = (-distance / temp).softmax(dim=-1) + per_sample_entropy = entropy(prob).mean(dim=[1, 2]) + avg_prob = reduce(prob, "... c d -> c d", "mean") + codebook_entropy = entropy(avg_prob).mean() + entropy_aux_loss = per_sample_entropy - codebook_entropy + + loss += self.entrop_loss_weight * entropy_aux_loss + + # TODO: return (indices, z_q, loss) + return ( + z_q, + loss.unsqueeze(1).unsqueeze(1).unsqueeze(1), + ( + indices, + self.commitment_loss_weight * commit_loss.mean().detach(), + self.entrop_loss_weight * entropy_aux_loss.mean().detach(), + self.entrop_loss_weight * per_sample_entropy.mean().detach(), + self.entrop_loss_weight * codebook_entropy.mean().detach(), + ), + ) + else: + return ( + z_q, + loss.unsqueeze(1).unsqueeze(1).unsqueeze(1), + self.commitment_loss_weight * commit_loss.mean().detach(), + ) + + +class InvQuantizerJit(nn.Module): + """Use for decoder_jit to trace quantizer in discrete tokenizer""" + + def __init__(self, quantizer): + super().__init__() + self.quantizer = quantizer + + def forward(self, indices: torch.Tensor): + codes = self.quantizer.indices_to_codes(indices) + return codes.to(self.quantizer.dtype) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/utils.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/utils.py new file mode 100644 index 00000000..9fc92ba4 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/modules/utils.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""Shared utilities for the networks module.""" + +from typing import Any + +import torch +from einops import pack, rearrange, unpack + + +def time2batch(x: torch.Tensor) -> tuple[torch.Tensor, int]: + batch_size = x.shape[0] + return rearrange(x, "b c t h w -> (b t) c h w"), batch_size + + +def batch2time(x: torch.Tensor, batch_size: int) -> torch.Tensor: + return rearrange(x, "(b t) c h w -> b c t h w", b=batch_size) + + +def space2batch(x: torch.Tensor) -> tuple[torch.Tensor, int]: + batch_size, height = x.shape[0], x.shape[-2] + return rearrange(x, "b c t h w -> (b h w) c t"), batch_size, height + + +def batch2space(x: torch.Tensor, batch_size: int, height: int) -> torch.Tensor: + return rearrange(x, "(b h w) c t -> b c t h w", b=batch_size, h=height) + + +def cast_tuple(t: Any, length: int = 1) -> Any: + return t if isinstance(t, tuple) else ((t,) * length) + + +def replication_pad(x): + return torch.cat([x[:, :, :1, ...], x], dim=2) + + +def divisible_by(num: int, den: int) -> bool: + return (num % den) == 0 + + +def is_odd(n: int) -> bool: + return not divisible_by(n, 2) + + +def nonlinearity(x): + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True) + + +class CausalNormalize(torch.nn.Module): + def __init__(self, in_channels, num_groups=1): + super().__init__() + self.norm = torch.nn.GroupNorm( + num_groups=num_groups, + num_channels=in_channels, + eps=1e-6, + affine=True, + ) + self.num_groups = num_groups + + def forward(self, x): + # if num_groups !=1, we apply a spatio-temporal groupnorm for backward compatibility purpose. + # All new models should use num_groups=1, otherwise causality is not guaranteed. + if self.num_groups == 1: + x, batch_size = time2batch(x) + return batch2time(self.norm(x), batch_size) + return self.norm(x) + + +def exists(v): + return v is not None + + +def default(*args): + for arg in args: + if exists(arg): + return arg + return None + + +def pack_one(t, pattern): + return pack([t], pattern) + + +def unpack_one(t, ps, pattern): + return unpack(t, ps, pattern)[0] + + +def round_ste(z: torch.Tensor) -> torch.Tensor: + """Round with straight through gradients.""" + zhat = z.round() + return z + (zhat - z).detach() + + +def log(t, eps=1e-5): + return t.clamp(min=eps).log() + + +def entropy(prob): + return (-prob * log(prob)).sum(dim=-1) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/README.md b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/README.md new file mode 100644 index 00000000..879eeb1b --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/README.md @@ -0,0 +1,179 @@ +# Cosmos Tokenizer: NeMo Framework Finetuning User Guide + +Post-train the Cosmos Tokenizer using the [NVIDIA NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html) to more accurately model previously unseen scenarios in your customer data, particularly for self-driving applications. By adapting the Cosmos Tokenizer to the specific characteristics and complexities of your in-house video content, you equip it to handle unique visual and temporal patterns that may have been missed during its initial pre-training. This enhanced modeling capability is essential for downstream diffusion models, which rely on the Tokenizer’s output to generate realistic physical scenes—ultimately boosting the performance and safety of your self-driving car systems. + +## Model Support Matrix + +The NeMo Framework currently supports the following Cosmos Tokenizer models. Review the available models for post-training. + +| Model Name | Model Ckpt | +|-------------------------|----------------------------| +| Cosmos-1.0-Tokenizer-CV8x8x8 | [HF Download](https://huggingface.co/nvidia/Cosmos-1.0-Tokenizer-CV8x8x8) | +| Cosmos-1.0-Tokenizer-DV8x16x16 | [HF Download](https://huggingface.co/nvidia/Cosmos-1.0-Tokenizer-DV8x16x16) | + +For optimal performance, we recommend utilizing GPUs such as the H100-80GB or A100-80GB. +Note: Have a use case that would benefit from an alternative tokenizer? We'd love to hear from you. You can submit a request via a GitHub issue. + +## Post-Training Support Matrix + +Cosmos Tokenizer can be post-trained for a variety of Physical AI tasks. Review the following table for a list of available Physical AI post-training tasks: + +| Post-training Task | Support Status | +|-------------------------|--------------------| +| General post-training and validation | **Supported** | + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the 80G H100 or A100 to run the model(s). + - **Containerization Platform**: We recommend using NVIDIA [NeMo Docker](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo/tags) Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Clone the Cosmos Repository + +```bash +git clone git@github.com:NVIDIA/Cosmos.git +``` + +### 3. Start the Container + +The [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos Tokenizer models. + +Run the following command to download and start the container: + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidian/nemo:cosmos.1.0.2 bash + ``` + +### 4. Download Checkpoints + +Follow the links provided in the Model Support Matrix to download the Cosmos Tokenizer checkpoints from Hugging Face. Detailed instructions for the download process are available on the Hugging Face page. + + +## Post-train + +Post-training a Cosmos Tokenizer enables you to train the model to compress videos that are more specific to your Physical AI use case. + +There are 3 steps to post-trainig: preparing a dataset, preprocessing the data, and post-training the model. + +### 1. Prepare a Dataset + +The first step is to prepare your dataset. Organize your data into a folder containing multiple video tars, each contains MP4 format videos (preferably at least 720p resolution). The recommended folder structure is as follows: + +- `000000.tar` + - `1.mp4` + - `2.mp4` +- `000001.tar` + - `3.mp4` + - `4.mp4` + +Here, 000000.tar and 000001.tar represent separate shards, and you may include additional shards as needed. + +Next we need to index the webdataset with [energon](). Navigate to the dataset directory and run the following command: + +```bash +energon prepare . --num-workers 8 --shuffle-tars +``` + +Interactively select dataset type `ImageWebdataset` and specify the type `mp4`. Below is an example of the interactive setup: + +``` +# energon prepare . --num-workers 8 --shuffle-tars +Found 2925 tar files in total. The first and last ones are: +- 000000.tar +- 002924.tar +If you want to exclude some of them, cancel with ctrl+c and specify an exclude filter in the command line. +Please enter a desired train/val/test split like "0.5, 0.2, 0.3" or "8,1,1": 99,1,0 +Indexing shards [####################################] 2925/2925 +Sample 0, keys: + - mp4 +Sample 1, keys: + - mp4 +Found the following part types in the dataset: mp4 +Do you want to create a dataset.yaml interactively? [Y/n]: +The following sample types are available: +0. CaptioningSample +1. ImageClassificationSample +2. ImageSample +3. InterleavedSample +4. MultiChoiceVQASample +5. OCRSample +6. Sample +7. SimilarityInterleavedSample +8. TextSample +9. VQASample +10. VidQASample +11. Crude sample (plain dict for cooking) +Please enter a number to choose a class: 2 +The sample type you selected: + +@dataclass +class ImageSample(Sample): + """Sample type for an image, e.g. for image reconstruction.""" + + #: The input image tensor in the shape (C, H, W) + image: torch.Tensor + +Do you want to set a simple field_map[Y] (or write your own sample_loader [n])? [Y/n]: Y + +For each field, please specify the corresponding name in the WebDataset. +Available types in WebDataset: mp4 +Leave empty for skipping optional field +You may also access json fields e.g. by setting the field to: json[field][field] +You may also specify alternative fields e.g. by setting to: jpg,png +Please enter the field_map for ImageSample: +Please enter a webdataset field name for 'image' (): mp4 +Done +``` + + +### 3. Post-train the Model + +The third step is to post-train the Cosmos tokenizer using the NeMo Framework. + +#### Run the Post-training Script + +Complete the following steps to post-train the Cosmos tokenizer Cosmos-1.0-Tokenizer-CV8x8x8. + +1. Run the following command to post-train Cosmos-1.0-Tokenizer-CV8x8x8: + ```bash + export CKPT_PTH="" # ${HF_HOME}/hub/models--nvidia--Cosmos-1.0-Tokenizer-CV8x8x8/snapshots/01f87fd67cebc32f1a2fd9e99d4e9614a6b3743b + export DATA="" + + # Optionally, you can monitor training progress with Weights and Biases (wandb). + export WANDB_API_KEY="
" + export WANDB_PROJECT_NAME="cosmos-diffusion-nemo-post-training" + export WANDB_RUN_ID="cosmos_diffusion_7b_text2world" + + torchrun --nproc-per-node 8 cosmos1/models/tokenizer/nemo/train_tokenizer.py --yes \ + data.path=$DATA \ + model.jit_ckpt_pth=$CKPT_PTH \ + model.model="Cosmos-1.0-Tokenizer-CV8x8x8" + ``` + +##### Configurable Hyperparameters + +For a comprehensive list of configurable hyperparameters, please refer to the `train_tokenizer.py` script. The script supports four major configuration components: + +1. **model**: Select a model for post-training and pass the model checkpoint. +2. **data**: Define batch size and dataloader related hyper-parameters. +3. **trainer**: Define the training loop. +4. **optim**: Specify the post-training optimizer hyperparameters. + +You can configure any hyperparameter of these four components by setting the value in the launch script using the following format: + +```bash +model.jit_ckpt_pth= +trainer.max_epochs= +``` + +Adjust the values as needed to suit your training requirements. After a few hundred iterations, you should observe that the 'loss' reported in Weights & Biases (`wandb`) starts decreasing. + + +

+ Image description +

diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/assets/loss.png b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/assets/loss.png new file mode 100644 index 00000000..aa01c9f3 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/assets/loss.png differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/requirements.txt b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/requirements.txt new file mode 100644 index 00000000..ed335f06 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/requirements.txt @@ -0,0 +1,3 @@ +pip install megatron-energon==4.0.0 pyav +pip install git+https://github.com/NVIDIA/NeMo-Run.git +pip install moviepy==1.0.3 imageio diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/train_tokenizer.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/train_tokenizer.py new file mode 100644 index 00000000..eb22904c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/nemo/train_tokenizer.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +import pytorch_lightning as pl +import torch.distributed +import torch.utils.checkpoint +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.data.diffusion_energon_datamodule import DiffusionDataModule +from nemo.collections.physicalai.tokenizer.tokenizer_model import MASK_KEY, VIDEO_KEY, TokenizerModel +from nemo.collections.physicalai.tokenizer.train_tokenizer import ImageTaskEncoder +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.optim.pytorch import PytorchOptimizerModule +from nemo.lightning.pytorch.plugins import MegatronDataSampler +from nemo.utils.exp_manager import TimingCallback +from pytorch_lightning.loggers import WandbLogger +from torch.utils.data import DataLoader + + +class FakeDataset(torch.utils.data.Dataset): + def __init__(self): + super().__init__() + + def __len__(self): + return 100000000 + + def __getitem__(self, idx): + input_t = torch.randn([2, 3, 33, 256, 256], dtype=torch.bfloat16, device="cuda") + mask_t = torch.ones_like(input_t, requires_grad=False, dtype=torch.bfloat16, device="cuda") + return {VIDEO_KEY: input_t, MASK_KEY: mask_t} + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +class FakeDataModule(pl.LightningDataModule): + def __init__( + self, + seq_length: int = 2048, + micro_batch_size: int = 1, + global_batch_size: int = 8, + num_workers: int = 1, + pin_memory: bool = True, + use_train_split_for_val: bool = False, + task_encoder=None, + ) -> None: + super().__init__() + self.seq_length = seq_length + self.micro_batch_size = micro_batch_size + self.global_batch_size = global_batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + + self.data_sampler = MegatronDataSampler( + seq_len=self.seq_length, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + ) + + def setup(self, stage: str = "") -> None: + self._train_ds = FakeDataset() + + def train_dataloader(self): + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def val_dataloader(self): + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def _create_dataloader(self, dataset, **kwargs): + return DataLoader( + dataset, + num_workers=self.num_workers, + pin_memory=True, + persistent_workers=True, + collate_fn=dataset.collate_fn, + **kwargs, + ) + + +@run.cli.factory(target=llm.train) +def train_tokenizer() -> run.Partial: + return run.Partial( + llm.train, + model=run.Config( + TokenizerModel, + jit_ckpt_pth=None, + model="Cosmos-1.0-Tokenizer-CV8x8x8", + ), + data=run.Config( + DiffusionDataModule, + path=None, + task_encoder=run.Config(ImageTaskEncoder), + global_batch_size=8, + micro_batch_size=1, + num_workers=8, + ), + trainer=run.Config( + nl.Trainer, + devices="auto", + num_nodes=int(os.environ.get("SLURM_NNODES", 1)), + accelerator="gpu", + strategy="ddp_find_unused_parameters_true", + num_sanity_val_steps=0, + limit_val_batches=1, + val_check_interval=100, + max_epochs=10000, + precision="bf16", + logger=WandbLogger(project="cosmos-tokenizer") if "WANDB_API_KEY" in os.environ else None, + log_every_n_steps=1, + use_distributed_sampler=False, + callbacks=[ + run.Config( + ModelCheckpoint, + monitor="global_step", + filename="{global_step}", + every_n_train_steps=100, + save_top_k=3, + mode="max", + always_save_context=False, + save_context_on_train_end=False, + ), + run.Config(PreemptionCallback), + run.Config(TimingCallback), + ], + ), + optim=run.Config( + PytorchOptimizerModule, + optimizer_fn=run.Partial( + torch.optim.AdamW, + lr=1e-4, + betas=(0.5, 0.999), + eps=1e-8, + weight_decay=0.01, + fused=True, + ), + ), + tokenizer=None, + resume=run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + ), + model_transform=None, + ) + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=train_tokenizer) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/__init__.py new file mode 100644 index 00000000..67f3a600 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/__init__.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from enum import Enum + +from cosmos1.models.tokenizer.networks.configs import continuous_image_8x8 as continuous_image_8x8_dict +from cosmos1.models.tokenizer.networks.configs import continuous_image_8x8_lowres as continuous_image_8x8_lowres_dict +from cosmos1.models.tokenizer.networks.configs import continuous_image_16x16 as continuous_image_16x16_dict +from cosmos1.models.tokenizer.networks.configs import ( + continuous_image_16x16_lowres as continuous_image_16x16_lowres_dict, +) +from cosmos1.models.tokenizer.networks.configs import continuous_video_4x8x8 as continuous_video_4x8x8_dict +from cosmos1.models.tokenizer.networks.configs import ( + continuous_video_4x8x8_lowres as continuous_video_4x8x8_lowres_dict, +) +from cosmos1.models.tokenizer.networks.configs import continuous_video_8x8x8 as continuous_video_8x8x8_dict +from cosmos1.models.tokenizer.networks.configs import continuous_video_8x16x16 as continuous_video_8x16x16_dict +from cosmos1.models.tokenizer.networks.configs import discrete_image_8x8 as discrete_image_8x8_dict +from cosmos1.models.tokenizer.networks.configs import discrete_image_8x8_lowres as discrete_image_8x8_lowres_dict +from cosmos1.models.tokenizer.networks.configs import discrete_image_16x16 as cdiscrete_image_16x16_dict +from cosmos1.models.tokenizer.networks.configs import discrete_image_16x16_lowres as discrete_image_16x16_lowres_dict +from cosmos1.models.tokenizer.networks.configs import discrete_video_4x8x8 as discrete_video_4x8x8_dict +from cosmos1.models.tokenizer.networks.configs import discrete_video_4x8x8_lowres as discrete_video_4x8x8_lowres_dict +from cosmos1.models.tokenizer.networks.configs import discrete_video_8x8x8 as discrete_video_8x8x8_dict +from cosmos1.models.tokenizer.networks.configs import discrete_video_8x16x16 as discrete_video_8x16x16_dict +from cosmos1.models.tokenizer.networks.continuous_image import ContinuousImageTokenizer +from cosmos1.models.tokenizer.networks.continuous_video import CausalContinuousVideoTokenizer +from cosmos1.models.tokenizer.networks.discrete_image import DiscreteImageTokenizer +from cosmos1.models.tokenizer.networks.discrete_video import CausalDiscreteVideoTokenizer + + +class TokenizerConfigs(Enum): + """Continuous Image (CI) Tokenizer Configs""" + + # Cosmos-0.1-Tokenizer-CI8x8 + CI8x8 = continuous_image_8x8_dict + + # Cosmos-0.1-Tokenizer-CI16x16 + CI16x16 = continuous_image_16x16_dict + + # Cosmos-1.0-Tokenizer-CI8x8-LowRes + CI8x8_LowRes = continuous_image_8x8_lowres_dict + + # Cosmos-1.0-Tokenizer-CI16x16-LowRes + CI16x16_LowRes = continuous_image_16x16_lowres_dict + + """Discrete Image (DI) Tokenizer Configs""" + # Cosmos-0.1-Tokenizer-DI8x8 + DI8x8 = discrete_image_8x8_dict + + # Cosmos-0.1-Tokenizer-DI16x16 + DI16x16 = cdiscrete_image_16x16_dict + + # Cosmos-1.0-Tokenizer-DI8x8-LowRes + DI8x8_LowRes = discrete_image_8x8_lowres_dict + + # Cosmos-1.0-Tokenizer-DI16x16-LowRes + DI16x16_LowRes = discrete_image_16x16_lowres_dict + + """Causal Continuous Video (CV) Tokenizer Configs""" + # Cosmos-0.1-Tokenizer-CV4x8x8 + CV4x8x8 = continuous_video_4x8x8_dict + + # Cosmos-0.1-Tokenizer-CV8x8x8 and Cosmos-1.0-Tokenizer-CV8x8x8 + CV8x8x8 = continuous_video_8x8x8_dict + + # Cosmos-0.1-Tokenizer-CV8x16x16 + CV8x16x16 = continuous_video_8x16x16_dict + + # Cosmos-1.0-Tokenizer-CV4x8x8-LowRes + CV4x8x8_LowRes = continuous_video_4x8x8_lowres_dict + + """Causal Discrete Video (DV) Tokenizer Configs""" + # Cosmos-0.1-Tokenizer-DV4x8x8 + DV4x8x8 = discrete_video_4x8x8_dict + + # Cosmos-0.1-Tokenizer-DV8x8x8 + DV8x8x8 = discrete_video_8x8x8_dict + + # Cosmos-0.1-Tokenizer-DV8x16x16 and Cosmos-1.0-Tokenizer-DV8x16x16 + DV8x16x16 = discrete_video_8x16x16_dict + + # Cosmos-1.0-Tokenizer-DV4x8x8-LowRes + DV4x8x8_LowRes = discrete_video_4x8x8_lowres_dict + + +class TokenizerModels(Enum): + CI = ContinuousImageTokenizer + DI = DiscreteImageTokenizer + CV = CausalContinuousVideoTokenizer + DV = CausalDiscreteVideoTokenizer diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/configs.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/configs.py new file mode 100644 index 00000000..2743239d --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/configs.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The default image and video tokenizer configs.""" + +from cosmos1.models.tokenizer.modules import ( + ContinuousFormulation, + Decoder3DType, + DecoderType, + DiscreteQuantizer, + Encoder3DType, + EncoderType, +) + + +continuous_image = dict( + # The attention resolution for res blocks. + attn_resolutions=[32], + # The base number of channels. + channels=128, + # The channel multipler for each resolution. + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + # The spatial compression ratio. + spatial_compression=16, + # The number of layers in each res block. + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + # The output latent dimension (channels). + latent_channels=16, + # The encoder output channels just before sampling. + # Which is also the decoder's input channels. + z_channels=16, + # A factor over the z_channels, to get the total channels the encoder should output. + # For a VAE for instance, we want to output the mean and variance, so we need 2 * z_channels. + z_factor=1, + name="CI", + # What formulation to use, either "AE" or "VAE". + # Chose VAE here, since the pre-trained ckpt were of a VAE formulation. + formulation=ContinuousFormulation.AE.name, + # Specify type of encoder ["Default", "LiteVAE"] + encoder=EncoderType.Default.name, + # Specify type of decoder ["Default"] + decoder=DecoderType.Default.name, +) +continuous_image_8x8 = dict(continuous_image) +continuous_image_8x8["spatial_compression"] = 8 + +continuous_image_16x16 = dict(continuous_image) +continuous_image_16x16["spatial_compression"] = 16 + +continuous_image_8x8_lowres = dict(continuous_image) +continuous_image_8x8_lowres["patch_size"] = 2 +continuous_image_8x8_lowres["spatial_compression"] = 8 + +continuous_image_16x16_lowres = dict(continuous_image) +continuous_image_16x16_lowres["patch_size"] = 2 +continuous_image_16x16_lowres["spatial_compression"] = 16 + + +discrete_image = dict( + # The attention resolution for res blocks. + attn_resolutions=[32], + # The base number of channels. + channels=128, + # The channel multipler for each resolution. + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + # The spatial compression ratio. + spatial_compression=16, + # The number of layers in each res block. + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + # The encoder output channels just before sampling. + z_channels=256, + # A factor over the z_channels, to get the total channels the encoder should output. + # for discrete tokenization, often we directly use the vector, so z_factor=1. + z_factor=1, + # The quantizer of choice, VQ, LFQ, FSQ, or ResFSQ. + quantizer=DiscreteQuantizer.FSQ.name, + # The embedding dimension post-quantization, which is also the input channels of the decoder. + # Which is also the output + embedding_dim=6, + # The number of levels to use for fine-scalar quantization. + levels=[8, 8, 8, 5, 5, 5], + # The number of quantizers to use for residual fine-scalar quantization. + num_quantizers=4, + name="DI", + # Specify type of encoder ["Default", "LiteVAE"] + encoder=EncoderType.Default.name, + # Specify type of decoder ["Default"] + decoder=DecoderType.Default.name, +) +discrete_image_8x8 = dict(discrete_image) +discrete_image_8x8["spatial_compression"] = 8 + +discrete_image_16x16 = dict(discrete_image) +discrete_image_16x16["spatial_compression"] = 16 + +discrete_image_8x8_lowres = dict(discrete_image) +discrete_image_8x8_lowres["patch_size"] = 2 +discrete_image_8x8_lowres["spatial_compression"] = 8 + +discrete_image_16x16_lowres = dict(discrete_image) +discrete_image_16x16_lowres["patch_size"] = 2 +discrete_image_16x16_lowres["spatial_compression"] = 16 + +continuous_video = dict( + attn_resolutions=[32], + channels=128, + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + latent_channels=16, + z_channels=16, + z_factor=1, + num_groups=1, + legacy_mode=False, + spatial_compression=8, + temporal_compression=8, + formulation=ContinuousFormulation.AE.name, + encoder=Encoder3DType.FACTORIZED.name, + decoder=Decoder3DType.FACTORIZED.name, + name="CV", +) +continuous_video_4x8x8 = dict(continuous_video) +continuous_video_4x8x8["channels_mult"] = [2, 4] +continuous_video_4x8x8["temporal_compression"] = 4 +continuous_video_4x8x8["spatial_compression"] = 8 + +continuous_video_8x8x8 = dict(continuous_video) +continuous_video_8x8x8["temporal_compression"] = 8 +continuous_video_8x8x8["spatial_compression"] = 8 + +continuous_video_8x16x16 = dict(continuous_video) +continuous_video_8x16x16["temporal_compression"] = 8 +continuous_video_8x16x16["spatial_compression"] = 16 + +continuous_video_4x8x8_lowres = dict(continuous_video) +continuous_video_4x8x8_lowres["temporal_compression"] = 4 +continuous_video_4x8x8_lowres["spatial_compression"] = 8 +continuous_video_4x8x8_lowres["patch_size"] = 2 + + +discrete_video = dict( + attn_resolutions=[32], + channels=128, + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + z_channels=16, + z_factor=1, + num_groups=1, + legacy_mode=False, + spatial_compression=16, + temporal_compression=8, + quantizer=DiscreteQuantizer.FSQ.name, + embedding_dim=6, + levels=[8, 8, 8, 5, 5, 5], + encoder=Encoder3DType.FACTORIZED.name, + decoder=Decoder3DType.FACTORIZED.name, + name="DV", +) + +discrete_video_4x8x8 = dict(discrete_video) +discrete_video_4x8x8["temporal_compression"] = 4 +discrete_video_4x8x8["spatial_compression"] = 8 + +discrete_video_8x8x8 = dict(discrete_video) +discrete_video_8x8x8["temporal_compression"] = 8 +discrete_video_8x8x8["spatial_compression"] = 8 + +discrete_video_8x16x16 = dict(discrete_video) +discrete_video_8x16x16["temporal_compression"] = 8 +discrete_video_8x16x16["spatial_compression"] = 16 + +discrete_video_4x8x8_lowres = dict(discrete_video) +discrete_video_4x8x8_lowres["z_channels"] = 256 +discrete_video_4x8x8_lowres["temporal_compression"] = 4 +discrete_video_4x8x8_lowres["spatial_compression"] = 8 +discrete_video_4x8x8_lowres["patch_size"] = 2 diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/continuous_image.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/continuous_image.py new file mode 100644 index 00000000..b5c1ffe1 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/continuous_image.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The continuous image tokenizer with VAE or AE formulation for 2D data.""" + +from collections import OrderedDict, namedtuple + +import torch +from cosmos1.models.tokenizer.modules import ContinuousFormulation, DecoderType, EncoderType +from loguru import logger as logging +from torch import nn + + +NetworkEval = namedtuple("NetworkEval", ["reconstructions", "posteriors", "latent"]) + + +class ContinuousImageTokenizer(nn.Module): + def __init__(self, z_channels: int, z_factor: int, latent_channels: int, **kwargs) -> None: + super().__init__() + self.name = kwargs.get("name", "ContinuousImageTokenizer") + self.latent_channels = latent_channels + + encoder_name = kwargs.get("encoder", EncoderType.Default.name) + self.encoder = EncoderType[encoder_name].value(z_channels=z_factor * z_channels, **kwargs) + + decoder_name = kwargs.get("decoder", DecoderType.Default.name) + self.decoder = DecoderType[decoder_name].value(z_channels=z_channels, **kwargs) + + self.quant_conv = torch.nn.Conv2d(z_factor * z_channels, z_factor * latent_channels, 1) + self.post_quant_conv = torch.nn.Conv2d(latent_channels, z_channels, 1) + + formulation_name = kwargs.get("formulation", ContinuousFormulation.AE.name) + self.distribution = ContinuousFormulation[formulation_name].value() + logging.info(f"{self.name} based on {formulation_name} formulation, with {kwargs}.") + + num_parameters = sum(param.numel() for param in self.parameters()) + logging.info(f"model={self.name}, num_parameters={num_parameters:,}") + logging.info(f"z_channels={z_channels}, latent_channels={self.latent_channels}.") + + def encoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("encoder", self.encoder), + ("quant_conv", self.quant_conv), + ("distribution", self.distribution), + ] + ) + ) + + def decoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("post_quant_conv", self.post_quant_conv), + ("decoder", self.decoder), + ] + ) + ) + + def last_decoder_layer(self): + return self.decoder.conv_out + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + return self.distribution(moments) + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input) -> dict[str, torch.Tensor] | NetworkEval: + latent, posteriors = self.encode(input) + dec = self.decode(latent) + if self.training: + return dict(reconstructions=dec, posteriors=posteriors, latent=latent) + return NetworkEval(reconstructions=dec, posteriors=posteriors, latent=latent) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/continuous_video.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/continuous_video.py new file mode 100644 index 00000000..f0c64e05 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/continuous_video.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The causal continuous video tokenizer with VAE or AE formulation for 3D data..""" + +from collections import OrderedDict, namedtuple + +from cosmos1.models.tokenizer.modules import ContinuousFormulation, Decoder3DType, Encoder3DType +from cosmos1.models.tokenizer.modules.layers3d import CausalConv3d +from loguru import logger as logging +from torch import nn + + +NetworkEval = namedtuple("NetworkEval", ["reconstructions", "posteriors", "latent"]) + + +class CausalContinuousVideoTokenizer(nn.Module): + def __init__(self, z_channels: int, z_factor: int, latent_channels: int, **kwargs) -> None: + super().__init__() + self.name = kwargs.get("name", "CausalContinuousVideoTokenizer") + self.latent_channels = latent_channels + + encoder_name = kwargs.get("encoder", Encoder3DType.BASE.name) + self.encoder = Encoder3DType[encoder_name].value(z_channels=z_factor * z_channels, **kwargs) + decoder_name = kwargs.get("decoder", Decoder3DType.BASE.name) + self.decoder = Decoder3DType[decoder_name].value(z_channels=z_channels, **kwargs) + + self.quant_conv = CausalConv3d( + z_factor * z_channels, + z_factor * latent_channels, + kernel_size=1, + padding=0, + ) + self.post_quant_conv = CausalConv3d(latent_channels, z_channels, kernel_size=1, padding=0) + + formulation_name = kwargs.get("formulation", ContinuousFormulation.AE.name) + self.distribution = ContinuousFormulation[formulation_name].value() + logging.info(f"{self.name} based on {formulation_name} formulation, with {kwargs}.") + + num_parameters = sum(param.numel() for param in self.parameters()) + logging.info(f"model={self.name}, num_parameters={num_parameters:,}") + logging.info(f"z_channels={z_channels}, latent_channels={self.latent_channels}.") + + def encoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("encoder", self.encoder), + ("quant_conv", self.quant_conv), + ("distribution", self.distribution), + ] + ) + ) + + def decoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("post_quant_conv", self.post_quant_conv), + ("decoder", self.decoder), + ] + ) + ) + + def last_decoder_layer(self): + return self.decoder.conv_out + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + return self.distribution(moments) + + def decode(self, z): + z = self.post_quant_conv(z) + return self.decoder(z) + + def forward(self, input): + latent, posteriors = self.encode(input) + reconstructions = self.decode(latent) + if self.training: + return dict( + reconstructions=reconstructions, + posteriors=posteriors, + latent=latent, + ) + return NetworkEval( + reconstructions=reconstructions, + posteriors=posteriors, + latent=latent, + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/discrete_image.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/discrete_image.py new file mode 100644 index 00000000..c875137e --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/discrete_image.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The network definition for discrete image tokenization with VQ, LFQ, FSQ or ResidualFSQ.""" + +from collections import OrderedDict, namedtuple + +import torch +from cosmos1.models.tokenizer.modules import DecoderType, DiscreteQuantizer, EncoderType +from cosmos1.models.tokenizer.modules.quantizers import InvQuantizerJit +from loguru import logger as logging +from torch import nn + + +NetworkEval = namedtuple("NetworkEval", ["reconstructions", "quant_loss", "quant_info"]) + + +class DiscreteImageTokenizer(nn.Module): + def __init__(self, z_channels: int, embedding_dim: int, **kwargs) -> None: + super().__init__() + self.name = kwargs.get("name", "DiscreteImageTokenizer") + self.embedding_dim = embedding_dim + + encoder_name = kwargs.get("encoder", EncoderType.Default.name) + self.encoder = EncoderType[encoder_name].value(z_channels=z_channels, **kwargs) + + decoder_name = kwargs.get("decoder", DecoderType.Default.name) + self.decoder = DecoderType[decoder_name].value(z_channels=z_channels, **kwargs) + self.quant_conv = nn.Conv2d(z_channels, embedding_dim, 1) + self.post_quant_conv = nn.Conv2d(embedding_dim, z_channels, 1) + + quantizer_name = kwargs.get("quantizer", DiscreteQuantizer.RESFSQ.name) + if quantizer_name == DiscreteQuantizer.VQ.name: + assert "num_embeddings" in kwargs, f"`num_embeddings` must be provided for {quantizer_name}." + kwargs.update(dict(embedding_dim=embedding_dim)) + elif quantizer_name == DiscreteQuantizer.LFQ.name: + assert "codebook_size" in kwargs, f"`codebook_size` must be provided for {quantizer_name}." + assert "codebook_dim" in kwargs, f"`codebook_dim` must be provided for {quantizer_name}." + elif quantizer_name == DiscreteQuantizer.FSQ.name: + assert "levels" in kwargs, f"`levels` must be provided for {quantizer_name}." + elif quantizer_name == DiscreteQuantizer.RESFSQ.name: + assert "levels" in kwargs, f"`levels` must be provided for {quantizer_name}.name." + assert "num_quantizers" in kwargs, f"`num_quantizers` must be provided for {quantizer_name}." + self.quantizer = DiscreteQuantizer[quantizer_name].value(**kwargs) + logging.info(f"{self.name} based on {quantizer_name}-VAE, with {kwargs}.") + + num_parameters = sum(param.numel() for param in self.parameters()) + logging.info(f"model={self.name}, num_parameters={num_parameters:,}") + logging.info(f"z_channels={z_channels}, embedding_dim={self.embedding_dim}.") + + def to(self, *args, **kwargs): + setattr(self.quantizer, "dtype", kwargs.get("dtype", torch.bfloat16)) + return super(DiscreteImageTokenizer, self).to(*args, **kwargs) + + def encoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("encoder", self.encoder), + ("quant_conv", self.quant_conv), + ("quantizer", self.quantizer), + ] + ) + ) + + def decoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("inv_quant", InvQuantizerJit(self.quantizer)), + ("post_quant_conv", self.post_quant_conv), + ("decoder", self.decoder), + ] + ) + ) + + def last_decoder_layer(self): + return self.decoder.conv_out + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return self.quantizer(h) + + def decode(self, quant): + quant = self.post_quant_conv(quant) + return self.decoder(quant) + + def decode_code(self, code_b): + quant_b = self.quantizer.indices_to_codes(code_b) + quant_b = self.post_quant_conv(quant_b) + return self.decoder(quant_b) + + def forward(self, input): + quant_info, quant_codes, quant_loss = self.encode(input) + reconstructions = self.decode(quant_codes) + if self.training: + return dict( + reconstructions=reconstructions, + quant_loss=quant_loss, + quant_info=quant_info, + ) + return NetworkEval( + reconstructions=reconstructions, + quant_loss=quant_loss, + quant_info=quant_info, + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/discrete_video.py b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/discrete_video.py new file mode 100644 index 00000000..3e1e2bd9 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/networks/discrete_video.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The network definition for discrete video tokenizer with VQ, LFQ, FSQ or ResidualFSQ.""" + +from collections import OrderedDict, namedtuple + +import torch +from cosmos1.models.tokenizer.modules import Decoder3DType, DiscreteQuantizer, Encoder3DType +from cosmos1.models.tokenizer.modules.layers3d import CausalConv3d +from cosmos1.models.tokenizer.modules.quantizers import InvQuantizerJit +from loguru import logger as logging +from torch import nn + + +NetworkEval = namedtuple("NetworkEval", ["reconstructions", "quant_loss", "quant_info"]) + + +class CausalDiscreteVideoTokenizer(nn.Module): + def __init__(self, z_channels: int, z_factor: int, embedding_dim: int, **kwargs) -> None: + super().__init__() + self.name = kwargs.get("name", "CausalDiscreteVideoTokenizer") + self.embedding_dim = embedding_dim + + encoder_name = kwargs.get("encoder", Encoder3DType.BASE.name) + self.encoder = Encoder3DType[encoder_name].value(z_channels=z_factor * z_channels, **kwargs) + + decoder_name = kwargs.get("decoder", Decoder3DType.BASE.name) + self.decoder = Decoder3DType[decoder_name].value(z_channels=z_channels, **kwargs) + + self.quant_conv = CausalConv3d(z_factor * z_channels, embedding_dim, kernel_size=1, padding=0) + self.post_quant_conv = CausalConv3d(embedding_dim, z_channels, kernel_size=1, padding=0) + + quantizer_name = kwargs.get("quantizer", DiscreteQuantizer.RESFSQ.name) + if quantizer_name == DiscreteQuantizer.VQ.name: + assert "num_embeddings" in kwargs, f"`num_embeddings` must be provided for {quantizer_name}." + kwargs.update(dict(embedding_dim=embedding_dim)) + elif quantizer_name == DiscreteQuantizer.LFQ.name: + assert "codebook_size" in kwargs, f"`codebook_size` must be provided for {quantizer_name}." + assert "codebook_dim" in kwargs, f"`codebook_dim` must be provided for {quantizer_name}." + elif quantizer_name == DiscreteQuantizer.FSQ.name: + assert "levels" in kwargs, f"`levels` must be provided for {quantizer_name}." + elif quantizer_name == DiscreteQuantizer.RESFSQ.name: + assert "levels" in kwargs, f"`levels` must be provided for {quantizer_name}." + assert "num_quantizers" in kwargs, f"`num_quantizers` must be provided for {quantizer_name}." + self.quantizer = DiscreteQuantizer[quantizer_name].value(**kwargs) + logging.info(f"{self.name} based on {quantizer_name}-VAE, with {kwargs}.") + + num_parameters = sum(param.numel() for param in self.parameters()) + logging.info(f"model={self.name}, num_parameters={num_parameters:,}") + logging.info(f"z_channels={z_channels}, embedding_dim={self.embedding_dim}.") + + def to(self, *args, **kwargs): + setattr(self.quantizer, "dtype", kwargs.get("dtype", torch.bfloat16)) + return super(CausalDiscreteVideoTokenizer, self).to(*args, **kwargs) + + def encoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("encoder", self.encoder), + ("quant_conv", self.quant_conv), + ("quantizer", self.quantizer), + ] + ) + ) + + def decoder_jit(self): + return nn.Sequential( + OrderedDict( + [ + ("inv_quant", InvQuantizerJit(self.quantizer)), + ("post_quant_conv", self.post_quant_conv), + ("decoder", self.decoder), + ] + ) + ) + + def last_decoder_layer(self): + return self.decoder.conv_out + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return self.quantizer(h) + + def decode(self, quant): + quant = self.post_quant_conv(quant) + return self.decoder(quant) + + def decode_code(self, code_b): + quant_b = self.quantizer.indices_to_codes(code_b) + quant_b = self.post_quant_conv(quant_b) + return self.decoder(quant_b) + + def forward(self, input): + quant_info, quant_codes, quant_loss = self.encode(input) + reconstructions = self.decode(quant_codes) + if self.training: + return dict( + reconstructions=reconstructions, + quant_loss=quant_loss, + quant_info=quant_info, + ) + return NetworkEval( + reconstructions=reconstructions, + quant_loss=quant_loss, + quant_info=quant_info, + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/notebook/Image_Tokenization.ipynb b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/notebook/Image_Tokenization.ipynb new file mode 100644 index 00000000..88a13a5f --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/notebook/Image_Tokenization.ipynb @@ -0,0 +1,250 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "n3ryhkSfIEfl" + }, + "source": [ + "# Image Tokenization Using [NVIDIA Cosmos Tokenizer](https://github.com/NVIDIA/Cosmos/blob/main/cosmos1/models/tokenizer) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/nvidia/Cosmos/blob/main/cosmos1/models/tokenizer/notebook/Image_Tokenization.ipynb)\n", + "\n", + "The Jupyter Notebook example utilizes the **Cosmos-Tokenizer** pretrained models, which include Continuous Image (CI) tokenizers that transform images into continuous latents and Discrete Image (DI) tokenizers that transform images into discrete tokens. Both CI and DI tokenizers are available with compression rates of 8x8 and 16x16. For instance, **CI16x16** effectively downsizes both height and width by a factor of 16.\n", + "\n", + "Within the notebook, the `ImageTokenizer` class from the `cosmos_tokenizer.image_lib` module is employed to manage the encoder and decoder components of this model. The encoder compresses the input image into a condensed latent representation or discrete integers, while the decoder reconstructs the image from this latent representation or discrete integers.\n", + "\n", + "This instance of the Cosmos Tokenizer demonstrates its autoencoding capability: compressing an image into a smaller latent space and subsequently reconstructing it to its original form. This showcases the efficiency of image tokenization for tasks involving significant spatial compression during image reconstruction, a highly desirable feature for generative modeling.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5BkjyLTPLM6e" + }, + "source": [ + "This tutorial follows a simple, step-by-step approach, making it easy to understand and adapt.\n", + "\n", + "## Step 1: Clone the Cosmos Tokenizer Repository" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TEV88M9YG973" + }, + "outputs": [], + "source": [ + "!git clone https://github.com/NVIDIA/Cosmos.git" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AxOMEJpFL9QL" + }, + "source": [ + "## Step 2: Install **Cosmos-Tokenizer**\n", + "Before proceeding, ensure you have the **Cosmos Tokenizer** installed. If you cloned the repository in Step 1, use the following command to install it in editable mode:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XuwUR6HrIxD8" + }, + "outputs": [], + "source": [ + "# Step 2: # Install Cosmos and its Python dependencies.\n", + "import os\n", + "if os.path.exists(\"Cosmos\"):\n", + " os.chdir(\"Cosmos\")\n", + " %pip install -r requirements.txt\n", + "else:\n", + " print('Cosmos-Tokenizer is already installed.')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "id29RPiyMOtB" + }, + "source": [ + "## Step 3: Set Up Hugging Face API Token and Download Pretrained Models\n", + "\n", + "In this step, you'll configure the Hugging Face API token and download the pretrained model weights required for the **Cosmos Tokenizer**.\n", + "\n", + "1. **Ensure You Have a Hugging Face Account** \n", + " If you do not already have a Hugging Face account, follow these steps to create one and generate an API token:\n", + " - Go to the [Hugging Face website](https://huggingface.co/) and sign up for a free account.\n", + " - After logging in, navigate to your [Settings → Access Tokens](https://huggingface.co/settings/tokens).\n", + " - Click on \"New Token\" to generate an API token with the required permissions.\n", + "\n", + "2. **Set the Hugging Face Token** \n", + " Check if the Hugging Face token is already set in the environment variables. If not, you will be prompted to enter it manually. The token is essential to authenticate and access the Hugging Face models.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "joxcyOlnM7HQ" + }, + "outputs": [], + "source": [ + "# Check if the token is already set\n", + "if \"HUGGINGFACE_TOKEN\" not in os.environ:\n", + " os.environ[\"HUGGINGFACE_TOKEN\"] = input(\"Please enter your Hugging Face API token: \")\n", + "!git config --global credential.helper store" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Lq7MAQ9pGPH9" + }, + "outputs": [], + "source": [ + "from huggingface_hub import login, snapshot_download\n", + "import os\n", + "HUGGINGFACE_TOKEN = os.environ.get(\"HUGGINGFACE_TOKEN\")\n", + "login(token=HUGGINGFACE_TOKEN, add_to_git_credential=True)\n", + "model_names = [\n", + " \"Cosmos-0.1-Tokenizer-CI8x8\",\n", + " \"Cosmos-0.1-Tokenizer-CI16x16\",\n", + " \"Cosmos-0.1-Tokenizer-DI8x8\",\n", + " \"Cosmos-0.1-Tokenizer-DI16x16\",\n", + "]\n", + "for model_name in model_names:\n", + " hf_repo = \"nvidia/\" + model_name\n", + " local_dir = \"checkpoints/\" + model_name\n", + " os.makedirs(local_dir, exist_ok=True)\n", + " print(f\"downloading {model_name}...\")\n", + " snapshot_download(repo_id=hf_repo, local_dir=local_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ltZ-v2vzNv74" + }, + "source": [ + "## Step 4: Use Cosmos Tokenizer for Image Reconstruction\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 839 + }, + "id": "gZFPrGCBGwtC", + "outputId": "0df7efc4-7a40-4011-81a6-3c541ba1601f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Input image read from:\t /content/Cosmos-Tokenizer/test_data/image.png\n", + "Reconstruction saved:\t /content/Cosmos-Tokenizer/test_data/image_CI8x8.png\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
Input Image
\n", + "
\n", + "
Reconstructed Image
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# @title In this step, load the required checkpoints, and perform image reconstruction. {\"run\":\"auto\"}\n", + "import cv2\n", + "import numpy as np\n", + "import torch\n", + "\n", + "import importlib\n", + "from cosmos1.models.tokenizer.inference.image_lib import ImageTokenizer\n", + "import mediapy as media\n", + "\n", + "\n", + "# 1) Specify the model name, and the paths to the encoder/decoder checkpoints.\n", + "model_name = 'Cosmos-0.1-Tokenizer-CI8x8' # @param [\"Cosmos-0.1-Tokenizer-CI16x16\", \"Cosmos-0.1-Tokenizer-CI8x8\", \"Cosmos-0.1-Tokenizer-DI8x8\", \"Cosmos-0.1-Tokenizer-DI16x16\"]\n", + "\n", + "encoder_ckpt = f\"checkpoints/{model_name}/encoder.jit\"\n", + "decoder_ckpt = f\"checkpoints/{model_name}/decoder.jit\"\n", + "\n", + "# 2) Load or provide the image filename you want to tokenize & reconstruct.\n", + "input_filepath = \"cosmos1/models/tokenizer/test_data/image.png\"\n", + "\n", + "# 3) Read the image from disk (shape = H x W x 3 in BGR). Then convert to RGB.\n", + "input_image = media.read_image(input_filepath)[..., :3]\n", + "assert input_image.ndim == 3 and input_image.shape[2] == 3, \"Image must have shape H x W x 3\"\n", + "\n", + "# 4) Expand dimensions to B x H x W x C, since the ImageTokenizer expects a batch dimension\n", + "# in the input. (Batch size = 1 in this example.)\n", + "batched_input_image = np.expand_dims(input_image, axis=0)\n", + "\n", + "# 5) Create the ImageTokenizer instance with the encoder & decoder.\n", + "# - device=\"cuda\" uses the GPU\n", + "# - dtype=\"bfloat16\" expects Ampere or newer GPU (A100, RTX 30xx, etc.)\n", + "tokenizer = ImageTokenizer(\n", + " checkpoint_enc=encoder_ckpt,\n", + " checkpoint_dec=decoder_ckpt,\n", + " device=\"cuda\",\n", + " dtype=\"bfloat16\",\n", + ")\n", + "\n", + "# 6) Use the tokenizer to autoencode (encode & decode) the image.\n", + "# The output is a NumPy array with shape = B x H x W x C, range [0..255].\n", + "batched_output_image = tokenizer(batched_input_image)\n", + "\n", + "# 7) Extract the single image from the batch (index 0), convert to uint8.\n", + "output_image = batched_output_image[0]\n", + "\n", + "# 9) Save the reconstructed image to disk.\n", + "input_dir, input_filename = os.path.split(input_filepath)\n", + "filename, ext = os.path.splitext(input_filename)\n", + "output_filepath = f\"{input_dir}/{filename}_{model_name.split('-')[-1]}{ext}\"\n", + "media.write_image(output_filepath, output_image)\n", + "print(\"Input image read from:\\t\", f\"{os.getcwd()}/{input_filepath}\")\n", + "print(\"Reconstruction saved:\\t\", f\"{os.getcwd()}/{output_filepath}\")\n", + "\n", + "# 10) Visualization of the input image (left) and the reconstruction (right).\n", + "media.show_images([input_image, output_image], [\"Input Image\", \"Reconstructed Image\"])" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/test_data/image.png b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/test_data/image.png new file mode 100644 index 00000000..ac6f379e Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/test_data/image.png differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/test_data/video.mp4 b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/test_data/video.mp4 new file mode 100644 index 00000000..af06d655 Binary files /dev/null and b/nemo_vfm/physicalai/Cosmos/cosmos1/models/tokenizer/test_data/video.mp4 differ diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/__init__.py new file mode 100644 index 00000000..dac9a4d7 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/config.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/config.py new file mode 100644 index 00000000..b535ecd5 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/config.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from __future__ import annotations + +from typing import Any, TypeVar + +import attrs + +from cosmos1.utils.lazy_config import LazyDict +from cosmos1.utils.misc import Color + + +T = TypeVar("T") + + +def _is_attrs_instance(obj: object) -> bool: + """ + Helper function to check if an object is an instance of an attrs-defined class. + + Args: + obj: The object to check. + + Returns: + bool: True if the object is an instance of an attrs-defined class, False otherwise. + """ + return hasattr(obj, "__attrs_attrs__") + + +def make_freezable(cls: T) -> T: + """ + A decorator that adds the capability to freeze instances of an attrs-defined class. + + NOTE: This requires the wrapped attrs to be defined with attrs.define(slots=False) because we need + to hack on a "_is_frozen" attribute. + + This decorator enhances an attrs-defined class with the ability to be "frozen" at runtime. + Once an instance is frozen, its attributes cannot be changed. It also recursively freezes + any attrs-defined objects that are attributes of the class. + + Usage: + @make_freezable + @attrs.define(slots=False) + class MyClass: + attribute1: int + attribute2: str + + obj = MyClass(1, 'a') + obj.freeze() # Freeze the instance + obj.attribute1 = 2 # Raises AttributeError + + Args: + cls: The class to be decorated. + + Returns: + The decorated class with added freezing capability. + """ + + if not hasattr(cls, "__dict__"): + raise TypeError( + "make_freezable cannot be used with classes that do not define __dict__. Make sure that the wrapped " + "class was defined with `@attrs.define(slots=False)`" + ) + + original_setattr = cls.__setattr__ + + def setattr_override(self, key, value) -> None: # noqa: ANN001 + """ + Override __setattr__ to allow modifications during initialization + and prevent modifications once the instance is frozen. + """ + if hasattr(self, "_is_frozen") and self._is_frozen and key != "_is_frozen": + raise AttributeError("Cannot modify frozen instance") + original_setattr(self, key, value) # type: ignore + + cls.__setattr__ = setattr_override # type: ignore + + def freeze(self: object) -> None: + """ + Freeze the instance and all its attrs-defined attributes. + """ + for _, value in attrs.asdict(self, recurse=False).items(): + if _is_attrs_instance(value) and hasattr(value, "freeze"): + value.freeze() + self._is_frozen = True # type: ignore + + cls.freeze = freeze # type: ignore + + return cls + + +def _pretty_print_attrs_instance(obj: object, indent: int = 0, use_color: bool = False) -> str: + """ + Recursively pretty prints attrs objects with color. + """ + + assert attrs.has(obj.__class__) + + lines: list[str] = [] + for attribute in attrs.fields(obj.__class__): + value = getattr(obj, attribute.name) + if attrs.has(value.__class__): + if use_color: + lines.append(" " * indent + Color.cyan("* ") + Color.green(attribute.name) + ":") + else: + lines.append(" " * indent + "* " + attribute.name + ":") + lines.append(_pretty_print_attrs_instance(value, indent + 1, use_color)) + else: + if use_color: + lines.append( + " " * indent + Color.cyan("* ") + Color.green(attribute.name) + ": " + Color.yellow(value) + ) + else: + lines.append(" " * indent + "* " + attribute.name + ": " + str(value)) + return "\n".join(lines) + + +@make_freezable +@attrs.define(slots=False) +class JobConfig: + # Project name. + project: str = "" + # Experiment name. + group: str = "" + # Run/job name. + name: str = "" + + @property + def path(self) -> str: + return f"{self.project}/{self.group}/{self.name}" + + +@make_freezable +@attrs.define(slots=False) +class Config: + """Config for a job. + + See /README.md/Configuration System for more info. + """ + + # Model configs. + model: LazyDict + + # Training job configs. + job: JobConfig = attrs.field(factory=JobConfig) + + def to_dict(self) -> dict[str, Any]: + return attrs.asdict(self) + + def validate(self) -> None: + """Validate that the config has all required fields.""" + assert self.job.project != "", "Project name is required." + assert self.job.group != "", "Group name is required." + assert self.job.name != "", "Job name is required." diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/config_helper.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/config_helper.py new file mode 100644 index 00000000..a734a233 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/config_helper.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import importlib +import os +import pkgutil +import sys +from dataclasses import fields as dataclass_fields +from dataclasses import is_dataclass +from typing import Any, Dict, Optional + +import attr +import attrs +from hydra import compose, initialize +from hydra.core.config_store import ConfigStore +from omegaconf import DictConfig, OmegaConf + +from cosmos1.utils import log +from cosmos1.utils.config import Config + + +def is_attrs_or_dataclass(obj) -> bool: + """ + Check if the object is an instance of an attrs class or a dataclass. + + Args: + obj: The object to check. + + Returns: + bool: True if the object is an instance of an attrs class or a dataclass, False otherwise. + """ + return is_dataclass(obj) or attr.has(type(obj)) + + +def get_fields(obj): + """ + Get the fields of an attrs class or a dataclass. + + Args: + obj: The object to get fields from. Must be an instance of an attrs class or a dataclass. + + Returns: + list: A list of field names. + + Raises: + ValueError: If the object is neither an attrs class nor a dataclass. + """ + if is_dataclass(obj): + return [field.name for field in dataclass_fields(obj)] + elif attr.has(type(obj)): + return [field.name for field in attr.fields(type(obj))] + else: + raise ValueError("The object is neither an attrs class nor a dataclass.") + + +def override(config: Config, overrides: Optional[list[str]] = None) -> Config: + """ + :param config: the instance of class `Config` (usually from `make_config`) + :param overrides: list of overrides for config + :return: the composed instance of class `Config` + """ + # Store the class of the config for reconstruction after overriding. + # config_class = type(config) + + # Convert Config object to a DictConfig object + config_dict = attrs.asdict(config) + config_omegaconf = DictConfig(content=config_dict, flags={"allow_objects": True}) + # Enforce "--" separator between the script arguments and overriding configs. + if overrides: + if overrides[0] != "--": + raise ValueError('Hydra config overrides must be separated with a "--" token.') + overrides = overrides[1:] + # Use Hydra to handle overrides + cs = ConfigStore.instance() + cs.store(name="config", node=config_omegaconf) + with initialize(version_base=None): + config_omegaconf = compose(config_name="config", overrides=overrides) + OmegaConf.resolve(config_omegaconf) + + def config_from_dict(ref_instance: Any, kwargs: Any) -> Any: + """ + Construct an instance of the same type as ref_instance using the provided dictionary or data or unstructured data + + Args: + ref_instance: The reference instance to determine the type and fields when needed + kwargs: A dictionary of keyword arguments to use for constructing the new instance or primitive data or unstructured data + + Returns: + Any: A new instance of the same type as ref_instance constructed using the provided kwargs or the primitive data or unstructured data + + Raises: + AssertionError: If the fields do not match or if extra keys are found. + Exception: If there is an error constructing the new instance. + """ + is_type = is_attrs_or_dataclass(ref_instance) + if not is_type: + return kwargs + else: + ref_fields = set(get_fields(ref_instance)) + assert isinstance(kwargs, dict) or isinstance(kwargs, DictConfig), ( + "kwargs must be a dictionary or a DictConfig" + ) + keys = set(kwargs.keys()) + + # ref_fields must equal to or include all keys + extra_keys = keys - ref_fields + assert ref_fields == keys or keys.issubset(ref_fields), ( + f"Fields mismatch: {ref_fields} != {keys}. Extra keys found: {extra_keys} \n \t when constructing {type(ref_instance)} with {keys}" + ) + + resolved_kwargs: Dict[str, Any] = {} + for f in keys: + resolved_kwargs[f] = config_from_dict(getattr(ref_instance, f), kwargs[f]) + try: + new_instance = type(ref_instance)(**resolved_kwargs) + except Exception as e: + log.error(f"Error when constructing {type(ref_instance)} with {resolved_kwargs}") + log.error(e) + raise e + return new_instance + + config = config_from_dict(config, config_omegaconf) + + return config + + +def get_config_module(config_file: str) -> str: + if not config_file.endswith(".py"): + log.error("Config file cannot be specified as module.") + log.error("Please provide the path to the Python config file (relative to the Cosmos root).") + assert os.path.isfile(config_file), f"Cosmos config file ({config_file}) not found." + # Convert to importable module format. + config_module = config_file.replace("/", ".").replace(".py", "") + return config_module + + +def import_all_modules_from_package(package_path: str, reload: bool = False, skip_underscore: bool = True) -> None: + """ + Import all modules from the specified package path recursively. + + This function is typically used in conjunction with Hydra to ensure that all modules + within a specified package are imported, which is necessary for registering configurations. + + Example usage: + ```python + import_all_modules_from_package("cosmos1.models.diffusion.config.inference", reload=True, skip_underscore=False) + ``` + + Args: + package_path (str): The dotted path to the package from which to import all modules. + reload (bool): Flag to determine whether to reload modules if they're already imported. + skip_underscore (bool): If True, skips importing modules that start with an underscore. + """ + log.debug(f"{'Reloading' if reload else 'Importing'} all modules from package {package_path}") + package = importlib.import_module(package_path) + package_directory = package.__path__ + + def import_modules_recursively(directory: str, prefix: str) -> None: + """ + Recursively imports or reloads all modules in the given directory. + + Args: + directory (str): The file system path to the current package directory. + prefix (str): The module prefix (e.g., 'cosmos1.models.diffusion.config'). + """ + for _, module_name, is_pkg in pkgutil.iter_modules([directory]): + if skip_underscore and module_name.startswith("_"): + log.debug(f"Skipping module {module_name} as it starts with an underscore") + continue + + full_module_name = f"{prefix}.{module_name}" + log.debug(f"{'Reloading' if reload else 'Importing'} module {full_module_name}") + + if full_module_name in sys.modules and reload: + importlib.reload(sys.modules[full_module_name]) + else: + importlib.import_module(full_module_name) + + if is_pkg: + sub_package_directory = os.path.join(directory, module_name) + import_modules_recursively(sub_package_directory, full_module_name) + + for directory in package_directory: + import_modules_recursively(directory, package_path) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/device.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/device.py new file mode 100644 index 00000000..db486afa --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/device.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import os + +import pynvml + + +class Device: + """A class to handle NVIDIA GPU device operations using NVML. + + This class provides an interface to access and manage NVIDIA GPU devices, + including retrieving device information and CPU affinity settings. + + Attributes: + _nvml_affinity_elements (int): Number of 64-bit elements needed to represent CPU affinity + """ + + _nvml_affinity_elements = math.ceil(os.cpu_count() / 64) # type: ignore + + def __init__(self, device_idx: int): + """Initialize a Device instance for a specific GPU. + + Args: + device_idx (int): Index of the GPU device to manage + + Raises: + NVMLError: If the device cannot be found or initialized + """ + super().__init__() + self.handle = pynvml.nvmlDeviceGetHandleByIndex(device_idx) + + def get_cpu_affinity(self) -> list[int]: + """Get the CPU affinity mask for this GPU device. + + Retrieves the CPU affinity mask indicating which CPU cores are assigned + to this GPU device. The affinity is returned as a list of CPU core indices. + + Returns: + list[int]: List of CPU core indices that have affinity with this GPU + + Raises: + NVMLError: If the CPU affinity information cannot be retrieved + + Example: + >>> device = Device(0) + >>> device.get_cpu_affinity() + [0, 1, 2, 3] # Shows this GPU has affinity with CPU cores 0-3 + """ + affinity_string = "" + for j in pynvml.nvmlDeviceGetCpuAffinity(self.handle, Device._nvml_affinity_elements): + # assume nvml returns list of 64 bit ints + affinity_string = "{:064b}".format(j) + affinity_string + affinity_list = [int(x) for x in affinity_string] + affinity_list.reverse() # so core 0 is in 0th element of list + return [i for i, e in enumerate(affinity_list) if e != 0] diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/distributed.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/distributed.py new file mode 100644 index 00000000..f7789248 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/distributed.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from __future__ import annotations + +import collections +import collections.abc +import ctypes +import functools +import os +from datetime import timedelta +from typing import Any, Callable, Optional + +import pynvml +import torch +import torch.distributed as dist + +from cosmos1.utils import log +from cosmos1.utils.device import Device + + +def init() -> int | None: + """Initialize distributed training.""" + # Set GPU affinity. + pynvml.nvmlInit() + local_rank = int(os.getenv("LOCAL_RANK", 0)) + device = Device(local_rank) + os.sched_setaffinity(0, device.get_cpu_affinity()) + # Set up NCCL communication. + os.environ["TORCH_NCCL_BLOCKING_WAIT"] = "0" + os.environ["TORCH_NCCL_ASYNC_ERROR_HANDLING"] = "1" + if dist.is_available(): + if dist.is_initialized(): + return torch.cuda.current_device() + torch.cuda.set_device(local_rank) + # Get the timeout value from environment variable + timeout_seconds = os.getenv("TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC", 1800) + # Convert the timeout to an integer (if it isn't already) and then to a timedelta + timeout_timedelta = timedelta(seconds=int(timeout_seconds)) + dist.init_process_group(backend="nccl", init_method="env://", timeout=timeout_timedelta) + log.critical( + f"Initialized distributed training with local rank {local_rank} with timeout {timeout_seconds}", + rank0_only=False, + ) + # Increase the L2 fetch granularity for faster speed. + _libcudart = ctypes.CDLL("libcudart.so") + # Set device limit on the current device. + p_value = ctypes.cast((ctypes.c_int * 1)(), ctypes.POINTER(ctypes.c_int)) + _libcudart.cudaDeviceSetLimit(ctypes.c_int(0x05), ctypes.c_int(128)) + _libcudart.cudaDeviceGetLimit(p_value, ctypes.c_int(0x05)) + log.info(f"Training with {get_world_size()} GPUs.") + + +def get_rank(group: Optional[dist.ProcessGroup] = None) -> int: + """Get the rank (GPU device) of the worker. + + Returns: + rank (int): The rank of the worker. + """ + rank = 0 + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank(group) + return rank + + +def get_world_size(group: Optional[dist.ProcessGroup] = None) -> int: + """Get world size. How many GPUs are available in this job. + + Returns: + world_size (int): The total number of GPUs available in this job. + """ + world_size = 1 + if dist.is_available() and dist.is_initialized(): + world_size = dist.get_world_size(group) + return world_size + + +def is_rank0() -> bool: + """Check if current process is the master GPU. + + Returns: + (bool): True if this function is called from the master GPU, else False. + """ + return get_rank() == 0 + + +def rank0_only(func: Callable) -> Callable: + """Apply this function only to the master GPU. + + Example usage: + @rank0_only + def func(x): + return x + 3 + + Args: + func (Callable): a function. + + Returns: + (Callable): A function wrapper executing the function only on the master GPU. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): # noqa: ANN202 + if is_rank0(): + return func(*args, **kwargs) + else: + return None + + return wrapper + + +def barrier() -> None: + """Barrier for all GPUs.""" + if dist.is_available() and dist.is_initialized(): + dist.barrier() + + +class DistributedDataParallel(torch.nn.parallel.DistributedDataParallel): + """This extends torch.nn.parallel.DistributedDataParallel with .training_step(). + + This borrows the concept of `forward-redirection` from Pytorch lightning. It wraps an coreModel such that + model.training_step() would be executed when calling self.training_step(), while preserving the behavior of calling + model() for Pytorch modules. Internally, this is a double rerouting mechanism (training_step -> forward -> + training_step), allowing us to preserve the function names and signatures. + """ + + def __init__(self, model: torch.nn.Module, *args, **kwargs): + super().__init__(model, *args, **kwargs) + + def training_step(self, *args, **kwargs) -> Any: + # Cache the original model.forward() method. + original_forward = self.module.forward + + def wrapped_training_step(*_args, **_kwargs): # noqa: ANN202 + # Unpatch immediately before calling training_step() because itself may want to call the real forward. + self.module.forward = original_forward + # The actual .training_step(). + return self.module.training_step(*_args, **_kwargs) + + # Patch the original_module's forward so we can redirect the arguments back to the real method. + self.module.forward = wrapped_training_step + # Call self, which implicitly calls self.forward() --> model.forward(), which is now model.training_step(). + # Without calling self.forward() or model.forward() explciitly, implicit hooks are also executed. + return self(*args, **kwargs) + + +def collate_batches(data_batches: list[dict[str, torch.Tensor]]) -> torch.Tensor | dict[str, torch.Tensor]: + """Aggregate the list of data batches from all devices and process the results. + + This is used for gathering validation data batches with utils.dataloader.DistributedEvalSampler. + It will return the data/output of the entire validation set in its original index order. The sizes of data_batches + in different ranks may differ by 1 (if dataset size is not evenly divisible), in which case a dummy sample will be + created before calling dis.all_gather(). + + Args: + data_batches (list[dict[str, torch.Tensor]]): List of tensors or (hierarchical) dictionary where + leaf entries are tensors. + + Returns: + data_gather (torch.Tensor | dict[str, torch.Tensor]): tensors or (hierarchical) dictionary where + leaf entries are concatenated tensors. + """ + if isinstance(data_batches[0], torch.Tensor): + # Concatenate the local data batches. + data_concat = torch.cat(data_batches, dim=0) # type: ignore + # Get the largest number of local samples from all ranks to determine whether to dummy-pad on this rank. + max_num_local_samples = torch.tensor(len(data_concat), device="cuda") + dist.all_reduce(max_num_local_samples, op=dist.ReduceOp.MAX) + if len(data_concat) < max_num_local_samples: + assert len(data_concat) + 1 == max_num_local_samples + dummy = torch.empty_like(data_concat[:1]) + data_concat = torch.cat([data_concat, dummy], dim=0) + dummy_count = torch.tensor(1, device="cuda") + else: + dummy_count = torch.tensor(0, device="cuda") + # Get all concatenated batches from all ranks and concatenate again. + dist.all_reduce(dummy_count, op=dist.ReduceOp.SUM) + data_concat = all_gather_tensor(data_concat.contiguous()) + data_collate = torch.stack(data_concat, dim=1).flatten(start_dim=0, end_dim=1) + # Remove the dummy samples. + if dummy_count > 0: + data_collate = data_collate[:-dummy_count] + elif isinstance(data_batches[0], collections.abc.Mapping): + data_collate = dict() + for key in data_batches[0].keys(): + data_collate[key] = collate_batches([data[key] for data in data_batches]) # type: ignore + else: + raise TypeError + return data_collate + + +@torch.no_grad() +def all_gather_tensor(tensor: torch.Tensor) -> list[torch.Tensor]: + """Gather the corresponding tensor from all GPU devices to a list. + + Args: + tensor (torch.Tensor): Pytorch tensor. + + Returns: + tensor_list (list[torch.Tensor]): A list of Pytorch tensors gathered from all GPU devices. + """ + tensor_list = [torch.zeros_like(tensor) for _ in range(get_world_size())] + dist.all_gather(tensor_list, tensor) + return tensor_list + + +def broadcast(tensor, src, group=None, async_op=False): + world_size = get_world_size() + if world_size < 2: + return tensor + dist.broadcast(tensor, src=src, group=group, async_op=async_op) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/io.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/io.py new file mode 100644 index 00000000..c877aa41 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/io.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from io import BytesIO +from typing import Dict, List + +import imageio +import numpy as np + + +def read_prompts_from_file(prompt_file: str) -> List[Dict[str, str]]: + """Read prompts from a JSONL file where each line is a dict with 'prompt' key and optionally 'visual_input' key. + + Args: + prompt_file (str): Path to JSONL file containing prompts + + Returns: + List[Dict[str, str]]: List of prompt dictionaries + """ + prompts = [] + with open(prompt_file, "r") as f: + for line in f: + prompt_dict = json.loads(line.strip()) + prompts.append(prompt_dict) + return prompts + + +def save_video(video, fps, H, W, video_save_quality, video_save_path): + """Save video frames to file. + + Args: + grid (np.ndarray): Video frames array [T,H,W,C] + fps (int): Frames per second + H (int): Frame height + W (int): Frame width + video_save_quality (int): Video encoding quality (0-10) + video_save_path (str): Output video file path + """ + kwargs = { + "fps": fps, + "quality": video_save_quality, + "macro_block_size": 1, + "ffmpeg_params": ["-s", f"{W}x{H}"], + "output_params": ["-f", "mp4"], + } + imageio.mimsave(video_save_path, video, "mp4", **kwargs) + + +def load_from_fileobj(filepath: str, format: str = "mp4", mode: str = "rgb", **kwargs): + """ + Load video from a file-like object using imageio with specified format and color mode. + + Parameters: + file (IO[bytes]): A file-like object containing video data. + format (str): Format of the video file (default 'mp4'). + mode (str): Color mode of the video, 'rgb' or 'gray' (default 'rgb'). + + Returns: + tuple: A tuple containing an array of video frames and metadata about the video. + """ + with open(filepath, "rb") as f: + value = f.read() + with BytesIO(value) as f: + f.seek(0) + video_reader = imageio.get_reader(f, format, **kwargs) + + video_frames = [] + for frame in video_reader: + if mode == "gray": + import cv2 # Convert frame to grayscale if mode is gray + + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) + frame = np.expand_dims(frame, axis=2) # Keep frame dimensions consistent + video_frames.append(frame) + + return np.array(video_frames), video_reader.get_meta_data() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/__init__.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/__init__.py new file mode 100644 index 00000000..762d1657 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/__init__.py @@ -0,0 +1,61 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import os + +from omegaconf import DictConfig, OmegaConf + +from cosmos1.utils.lazy_config.instantiate import instantiate +from cosmos1.utils.lazy_config.lazy import LazyCall, LazyConfig +from cosmos1.utils.lazy_config.omegaconf_patch import to_object + + +OmegaConf.to_object = to_object + +PLACEHOLDER = None +LazyDict = DictConfig + +__all__ = ["instantiate", "LazyCall", "LazyConfig", "PLACEHOLDER", "LazyDict"] + + +DOC_BUILDING = os.getenv("_DOC_BUILDING", False) # set in docs/conf.py + + +def fixup_module_metadata(module_name, namespace, keys=None): + """ + Fix the __qualname__ of module members to be their exported api name, so + when they are referenced in docs, sphinx can find them. Reference: + https://github.com/python-trio/trio/blob/6754c74eacfad9cc5c92d5c24727a2f3b620624e/trio/_util.py#L216-L241 + """ + if not DOC_BUILDING: + return + seen_ids = set() + + def fix_one(qualname, name, obj): + # avoid infinite recursion (relevant when using + # typing.Generic, for example) + if id(obj) in seen_ids: + return + seen_ids.add(id(obj)) + + mod = getattr(obj, "__module__", None) + if mod is not None and (mod.startswith(module_name) or mod.startswith("fvcore.")): + obj.__module__ = module_name + # Modules, unlike everything else in Python, put fully-qualitied + # names into their __name__ attribute. We check for "." to avoid + # rewriting these. + if hasattr(obj, "__name__") and "." not in obj.__name__: + obj.__name__ = name + obj.__qualname__ = qualname + if isinstance(obj, type): + for attr_name, attr_value in obj.__dict__.items(): + fix_one(objname + "." + attr_name, attr_name, attr_value) + + if keys is None: + keys = namespace.keys() + for objname in keys: + if not objname.startswith("_"): + obj = namespace[objname] + fix_one(objname, objname, obj) + + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/file_io.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/file_io.py new file mode 100644 index 00000000..b883a4c0 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/file_io.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler +from iopath.common.file_io import PathManager as PathManagerBase + + +__all__ = ["PathManager", "PathHandler"] + + +PathManager = PathManagerBase() +PathManager.register_handler(HTTPURLHandler()) +PathManager.register_handler(OneDrivePathHandler()) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/instantiate.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/instantiate.py new file mode 100644 index 00000000..7ff24804 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/instantiate.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import collections.abc as abc +import dataclasses +import logging +from typing import Any + +import attrs + +from cosmos1.utils.lazy_config.registry import _convert_target_to_string, locate + + +__all__ = ["dump_dataclass", "instantiate"] + + +def is_dataclass_or_attrs(target): + return dataclasses.is_dataclass(target) or attrs.has(target) + + +def dump_dataclass(obj: Any): + """ + Dump a dataclass recursively into a dict that can be later instantiated. + + Args: + obj: a dataclass object + + Returns: + dict + """ + assert dataclasses.is_dataclass(obj) and not isinstance(obj, type), ( + "dump_dataclass() requires an instance of a dataclass." + ) + ret = {"_target_": _convert_target_to_string(type(obj))} + for f in dataclasses.fields(obj): + v = getattr(obj, f.name) + if dataclasses.is_dataclass(v): + v = dump_dataclass(v) + if isinstance(v, (list, tuple)): + v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v] + ret[f.name] = v + return ret + + +def instantiate(cfg, *args, **kwargs): + """ + Recursively instantiate objects defined in dictionaries by + "_target_" and arguments. + + Args: + cfg: a dict-like object with "_target_" that defines the caller, and + other keys that define the arguments + args: Optional positional parameters pass-through. + kwargs: Optional named parameters pass-through. + + Returns: + object instantiated by cfg + """ + from omegaconf import DictConfig, ListConfig, OmegaConf + + if isinstance(cfg, ListConfig): + lst = [instantiate(x) for x in cfg] + return ListConfig(lst, flags={"allow_objects": True}) + if isinstance(cfg, list): + # Specialize for list, because many classes take + # list[objects] as arguments, such as ResNet, DatasetMapper + return [instantiate(x) for x in cfg] + + # If input is a DictConfig backed by dataclasses (i.e. omegaconf's structured config), + # instantiate it to the actual dataclass. + if isinstance(cfg, DictConfig) and is_dataclass_or_attrs(cfg._metadata.object_type): + return OmegaConf.to_object(cfg) + + if isinstance(cfg, abc.Mapping) and "_target_" in cfg: + # conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all, + # but faster: https://github.com/facebookresearch/hydra/issues/1200 + cfg = {k: instantiate(v) for k, v in cfg.items()} + cls = cfg.pop("_target_") + cls = instantiate(cls) + + if isinstance(cls, str): + cls_name = cls + cls = locate(cls_name) + assert cls is not None, cls_name + else: + try: + cls_name = cls.__module__ + "." + cls.__qualname__ + except Exception: + # target could be anything, so the above could fail + cls_name = str(cls) + assert callable(cls), f"_target_ {cls} does not define a callable object" + try: + # override config with kwargs + instantiate_kwargs = {} + instantiate_kwargs.update(cfg) + instantiate_kwargs.update(kwargs) + return cls(*args, **instantiate_kwargs) + except TypeError: + logger = logging.getLogger(__name__) + logger.error(f"Error when instantiating {cls_name}!") + raise + return cfg # return as-is if don't know what to do diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/lazy.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/lazy.py new file mode 100644 index 00000000..5c643902 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/lazy.py @@ -0,0 +1,275 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import ast +import builtins +import collections.abc as abc +import importlib +import inspect +import os +import uuid +from collections import OrderedDict +from contextlib import contextmanager +from dataclasses import is_dataclass +from typing import Any, Dict, List, Tuple, Union + +import attrs +import yaml +from omegaconf import DictConfig, ListConfig, OmegaConf + +from cosmos1.utils.lazy_config.file_io import PathManager +from cosmos1.utils.lazy_config.registry import _convert_target_to_string + + +__all__ = ["LazyCall", "LazyConfig"] + + +def sort_dict(d: Dict[str, Any]) -> OrderedDict[str, Any]: + return OrderedDict(sorted(d.items(), key=lambda x: x[0])) + + +def dict_representer(dumper: yaml.Dumper, data: OrderedDict[str, Any]) -> yaml.nodes.MappingNode: + return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) + + +def sort_recursive(obj: Union[Dict[str, Any], List[Any], Any]) -> Union[OrderedDict[str, Any], List[Any], Any]: + if isinstance(obj, dict): + return sort_dict({k: sort_recursive(v) for k, v in obj.items()}) + elif isinstance(obj, list): + return [sort_recursive(item) for item in obj] + return obj + + +yaml.add_representer(OrderedDict, dict_representer) + + +def get_default_params(cls_or_func): + if callable(cls_or_func): + # inspect signature for function + signature = inspect.signature(cls_or_func) + else: + # inspect signature for class + signature = inspect.signature(cls_or_func.__init__) + params = signature.parameters + default_params = { + name: param.default for name, param in params.items() if param.default is not inspect.Parameter.empty + } + return default_params + + +class LazyCall: + """ + Wrap a callable so that when it's called, the call will not be executed, + but returns a dict that describes the call. + + LazyCall object has to be called with only keyword arguments. Positional + arguments are not yet supported. + + Examples: + :: + from detectron2.config import instantiate, LazyCall + + layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32) + layer_cfg.out_channels = 64 # can edit it afterwards + layer = instantiate(layer_cfg) + """ + + def __init__(self, target): + if not (callable(target) or isinstance(target, (str, abc.Mapping))): + raise TypeError(f"target of LazyCall must be a callable or defines a callable! Got {target}") + self._target = target + + def __call__(self, **kwargs): + if is_dataclass(self._target) or attrs.has(self._target): + # omegaconf object cannot hold dataclass type + # https://github.com/omry/omegaconf/issues/784 + target = _convert_target_to_string(self._target) + else: + target = self._target + kwargs["_target_"] = target + + _final_params = get_default_params(self._target) + _final_params.update(kwargs) + + return DictConfig(content=_final_params, flags={"allow_objects": True}) + + +def _visit_dict_config(cfg, func): + """ + Apply func recursively to all DictConfig in cfg. + """ + if isinstance(cfg, DictConfig): + func(cfg) + for v in cfg.values(): + _visit_dict_config(v, func) + elif isinstance(cfg, ListConfig): + for v in cfg: + _visit_dict_config(v, func) + + +def _validate_py_syntax(filename): + # see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py + with PathManager.open(filename, "r") as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError(f"Config file {filename} has syntax error!") from e + + +def _cast_to_config(obj): + # if given a dict, return DictConfig instead + if isinstance(obj, dict): + return DictConfig(obj, flags={"allow_objects": True}) + return obj + + +_CFG_PACKAGE_NAME = "detectron2._cfg_loader" +""" +A namespace to put all imported config into. +""" + + +def _random_package_name(filename): + # generate a random package name when loading config files + return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename) + + +@contextmanager +def _patch_import(): + """ + Enhance relative import statements in config files, so that they: + 1. locate files purely based on relative location, regardless of packages. + e.g. you can import file without having __init__ + 2. do not cache modules globally; modifications of module states has no side effect + 3. support other storage system through PathManager, so config files can be in the cloud + 4. imported dict are turned into omegaconf.DictConfig automatically + """ + old_import = builtins.__import__ + + def find_relative_file(original_file, relative_import_path, level): + # NOTE: "from . import x" is not handled. Because then it's unclear + # if such import should produce `x` as a python module or DictConfig. + # This can be discussed further if needed. + relative_import_err = """ +Relative import of directories is not allowed within config files. +Within a config file, relative import can only import other config files. +""".replace("\n", " ") + if not len(relative_import_path): + raise ImportError(relative_import_err) + + cur_file = os.path.dirname(original_file) + for _ in range(level - 1): + cur_file = os.path.dirname(cur_file) + cur_name = relative_import_path.lstrip(".") + for part in cur_name.split("."): + cur_file = os.path.join(cur_file, part) + if not cur_file.endswith(".py"): + cur_file += ".py" + if not PathManager.isfile(cur_file): + cur_file_no_suffix = cur_file[: -len(".py")] + if PathManager.isdir(cur_file_no_suffix): + raise ImportError(f"Cannot import from {cur_file_no_suffix}." + relative_import_err) + else: + raise ImportError( + f"Cannot import name {relative_import_path} from {original_file}: {cur_file} does not exist." + ) + return cur_file + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + if ( + # Only deal with relative imports inside config files + level != 0 and globals is not None and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME) + ): + cur_file = find_relative_file(globals["__file__"], name, level) + _validate_py_syntax(cur_file) + spec = importlib.machinery.ModuleSpec(_random_package_name(cur_file), None, origin=cur_file) + module = importlib.util.module_from_spec(spec) + module.__file__ = cur_file + with PathManager.open(cur_file) as f: + content = f.read() + exec(compile(content, cur_file, "exec"), module.__dict__) + for name in fromlist: # turn imported dict into DictConfig automatically + val = _cast_to_config(module.__dict__[name]) + module.__dict__[name] = val + return module + return old_import(name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + yield new_import + builtins.__import__ = old_import + + +class LazyConfig: + """ + Provide methods to save, load, and overrides an omegaconf config object + which may contain definition of lazily-constructed objects. + """ + + @staticmethod + def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Load a config file. + + Args: + filename: absolute path or relative path w.r.t. the current working directory + keys: keys to load and return. If not given, return all keys + (whose values are config objects) in a dict. + """ + has_keys = keys is not None + filename = filename.replace("/./", "/") # redundant + if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]: + raise ValueError(f"Config file {filename} has to be a python or yaml file.") + if filename.endswith(".py"): + _validate_py_syntax(filename) + + with _patch_import(): + # Record the filename + module_namespace = { + "__file__": filename, + "__package__": _random_package_name(filename), + } + with PathManager.open(filename) as f: + content = f.read() + # Compile first with filename to: + # 1. make filename appears in stacktrace + # 2. make load_rel able to find its parent's (possibly remote) location + exec(compile(content, filename, "exec"), module_namespace) + + ret = module_namespace + else: + with PathManager.open(filename) as f: + obj = yaml.unsafe_load(f) + ret = OmegaConf.create(obj, flags={"allow_objects": True}) + + if has_keys: + if isinstance(keys, str): + return _cast_to_config(ret[keys]) + else: + return tuple(_cast_to_config(ret[a]) for a in keys) + else: + if filename.endswith(".py"): + # when not specified, only load those that are config objects + ret = DictConfig( + { + name: _cast_to_config(value) + for name, value in ret.items() + if isinstance(value, (DictConfig, ListConfig, dict)) and not name.startswith("_") + }, + flags={"allow_objects": True}, + ) + return ret diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/omegaconf_patch.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/omegaconf_patch.py new file mode 100644 index 00000000..39dca42a --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/omegaconf_patch.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Union + +from omegaconf import OmegaConf +from omegaconf.base import DictKeyType, SCMode +from omegaconf.dictconfig import DictConfig # pragma: no cover + + +def to_object(cfg: Any) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: + """ + Converts an OmegaConf configuration object to a native Python container (dict or list), unless + the configuration is specifically created by LazyCall, in which case the original configuration + is returned directly. + + This function serves as a modification of the original `to_object` method from OmegaConf, + preventing DictConfig objects created by LazyCall from being automatically converted to Python + dictionaries. This ensures that configurations meant to be lazily evaluated retain their intended + structure and behavior. + + Differences from OmegaConf's original `to_object`: + - Adds a check at the beginning to return the configuration unchanged if it is created by LazyCall. + + Reference: + - Original OmegaConf `to_object` method: https://github.com/omry/omegaconf/blob/master/omegaconf/omegaconf.py#L595 + + Args: + cfg (Any): The OmegaConf configuration object to convert. + + Returns: + Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: The converted Python container if + `cfg` is not a LazyCall created configuration, otherwise the unchanged `cfg`. + + Examples: + >>> cfg = DictConfig({"key": "value", "_target_": "Model"}) + >>> to_object(cfg) + DictConfig({"key": "value", "_target_": "Model"}) + + >>> cfg = DictConfig({"list": [1, 2, 3]}) + >>> to_object(cfg) + {'list': [1, 2, 3]} + """ + if isinstance(cfg, DictConfig) and "_target_" in cfg.keys(): + return cfg + + return OmegaConf.to_container( + cfg=cfg, + resolve=True, + throw_on_missing=True, + enum_to_str=False, + structured_config_mode=SCMode.INSTANTIATE, + ) diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/registry.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/registry.py new file mode 100644 index 00000000..bc264d7c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/lazy_config/registry.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pydoc +from typing import Any + + +""" +`locate` provide ways to map a string (typically found +in config files) to callable objects. +""" + +__all__ = ["locate"] + + +def _convert_target_to_string(t: Any) -> str: + """ + Inverse of ``locate()``. + + Args: + t: any object with ``__module__`` and ``__qualname__`` + """ + module, qualname = t.__module__, t.__qualname__ + + # Compress the path to this object, e.g. ``module.submodule._impl.class`` + # may become ``module.submodule.class``, if the later also resolves to the same + # object. This simplifies the string, and also is less affected by moving the + # class implementation. + module_parts = module.split(".") + for k in range(1, len(module_parts)): + prefix = ".".join(module_parts[:k]) + candidate = f"{prefix}.{qualname}" + try: + if locate(candidate) is t: + return candidate + except ImportError: + pass + return f"{module}.{qualname}" + + +def locate(name: str) -> Any: + """ + Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``, + such as "module.submodule.class_name". + + Raise Exception if it cannot be found. + """ + obj = pydoc.locate(name) + + # Some cases (e.g. torch.optim.sgd.SGD) not handled correctly + # by pydoc.locate. Try a private function from hydra. + if obj is None: + try: + # from hydra.utils import get_method - will print many errors + from hydra.utils import _locate + except ImportError as e: + raise ImportError(f"Cannot dynamically locate object {name}!") from e + else: + obj = _locate(name) # it raises if fails + + return obj diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/log.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/log.py new file mode 100644 index 00000000..6474813c --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/log.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import atexit +import os +import sys +from typing import Any, Optional + +import torch.distributed as dist +from loguru._logger import Core, Logger + + +RANK0_ONLY = True +LEVEL = os.environ.get("LOGURU_LEVEL", "INFO") + +logger = Logger( + core=Core(), + exception=None, + depth=1, + record=False, + lazy=False, + colors=False, + raw=False, + capture=True, + patchers=[], + extra={}, +) + +atexit.register(logger.remove) + + +def _add_relative_path(record: dict[str, Any]) -> None: + start = os.getcwd() + record["extra"]["relative_path"] = os.path.relpath(record["file"].path, start) + + +*options, _, extra = logger._options # type: ignore +logger._options = tuple([*options, [_add_relative_path], extra]) # type: ignore + + +def init_loguru_stdout() -> None: + logger.remove() + machine_format = get_machine_format() + message_format = get_message_format() + logger.add( + sys.stdout, + level=LEVEL, + format=f"[{{time:MM-DD HH:mm:ss}}|{machine_format}{message_format}", + filter=_rank0_only_filter, + ) + + +def get_machine_format() -> str: + node_id = os.environ.get("NGC_ARRAY_INDEX", "0") + num_nodes = int(os.environ.get("NGC_ARRAY_SIZE", "1")) + machine_format = "" + rank = 0 + if dist.is_available(): + if not RANK0_ONLY and dist.is_initialized(): + rank = dist.get_rank() + world_size = dist.get_world_size() + machine_format = ( + f"[Node{node_id:<3}/{num_nodes:<3}][RANK{rank:<5}/{world_size:<5}]" + + "[{process.name:<8}]| " + ) + return machine_format + + +def get_message_format() -> str: + message_format = "{level}|{extra[relative_path]}:{line}:{function}] {message}" + return message_format + + +def _rank0_only_filter(record: Any) -> bool: + is_rank0 = record["extra"].get("rank0_only", True) + if _get_rank() == 0 and is_rank0: + return True + if not is_rank0: + record["message"] = f"[RANK {_get_rank()}]" + record["message"] + return not is_rank0 + + +def trace(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).trace(message) + + +def debug(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).debug(message) + + +def info(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).info(message) + + +def success(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).success(message) + + +def warning(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).warning(message) + + +def error(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).error(message) + + +def critical(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).critical(message) + + +def exception(message: str, rank0_only: bool = True) -> None: + logger.opt(depth=1).bind(rank0_only=rank0_only).exception(message) + + +def _get_rank(group: Optional[dist.ProcessGroup] = None) -> int: + """Get the rank (GPU device) of the worker. + + Returns: + rank (int): The rank of the worker. + """ + rank = 0 + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank(group) + return rank + + +# Execute at import time. +init_loguru_stdout() diff --git a/nemo_vfm/physicalai/Cosmos/cosmos1/utils/misc.py b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/misc.py new file mode 100644 index 00000000..c8a22588 --- /dev/null +++ b/nemo_vfm/physicalai/Cosmos/cosmos1/utils/misc.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from __future__ import annotations + +import collections +import collections.abc +import functools +import json +import random +import time +from contextlib import ContextDecorator +from typing import Any, Callable, List, Tuple, TypeVar + +import numpy as np +import termcolor +import torch + +from cosmos1.utils import distributed, log + + +def to( + data: Any, + device: str | torch.device | None = None, + dtype: torch.dtype | None = None, + memory_format: torch.memory_format = torch.preserve_format, +) -> Any: + """Recursively cast data into the specified device, dtype, and/or memory_format. + + The input data can be a tensor, a list of tensors, a dict of tensors. + See the documentation for torch.Tensor.to() for details. + + Args: + data (Any): Input data. + device (str | torch.device): GPU device (default: None). + dtype (torch.dtype): data type (default: None). + memory_format (torch.memory_format): memory organization format (default: torch.preserve_format). + + Returns: + data (Any): Data cast to the specified device, dtype, and/or memory_format. + """ + assert device is not None or dtype is not None or memory_format is not None, ( + "at least one of device, dtype, memory_format should be specified" + ) + if isinstance(data, torch.Tensor): + is_cpu = (isinstance(device, str) and device == "cpu") or ( + isinstance(device, torch.device) and device.type == "cpu" + ) + data = data.to( + device=device, + dtype=dtype, + memory_format=memory_format, + non_blocking=(not is_cpu), + ) + return data + elif isinstance(data, collections.abc.Mapping): + return type(data)( + {key: to(data[key], device=device, dtype=dtype, memory_format=memory_format) for key in data} + ) + elif isinstance(data, collections.abc.Sequence) and not isinstance(data, (str, bytes)): + return type(data)([to(elem, device=device, dtype=dtype, memory_format=memory_format) for elem in data]) + else: + return data + + +def serialize(data: Any) -> Any: + """Serialize data by hierarchically traversing through iterables. + + Args: + data (Any): Input data. + + Returns: + data (Any): Serialized data. + """ + if isinstance(data, collections.abc.Mapping): + return type(data)({key: serialize(data[key]) for key in data}) + elif isinstance(data, collections.abc.Sequence) and not isinstance(data, (str, bytes)): + return type(data)([serialize(elem) for elem in data]) + else: + try: + json.dumps(data) + except TypeError: + data = str(data) + return data + + +def set_random_seed(seed: int, by_rank: bool = False) -> None: + """Set random seed. This includes random, numpy, Pytorch. + + Args: + seed (int): Random seed. + by_rank (bool): if true, each GPU will use a different random seed. + """ + if by_rank: + seed += distributed.get_rank() + log.info(f"Using random seed {seed}.") + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) # sets seed on the current CPU & all GPUs + + +def arch_invariant_rand( + shape: List[int] | Tuple[int], dtype: torch.dtype, device: str | torch.device, seed: int | None = None +): + """Produce a GPU-architecture-invariant randomized Torch tensor. + + Args: + shape (list or tuple of ints): Output tensor shape. + dtype (torch.dtype): Output tensor type. + device (torch.device): Device holding the output. + seed (int): Optional randomization seed. + + Returns: + tensor (torch.tensor): Randomly-generated tensor. + """ + # Create a random number generator, optionally seeded + rng = np.random.RandomState(seed) + + # # Generate random numbers using the generator + random_array = rng.standard_normal(shape).astype(np.float32) # Use standard_normal for normal distribution + + # Convert to torch tensor and return + return torch.from_numpy(random_array).to(dtype=dtype, device=device) + + +T = TypeVar("T", bound=Callable[..., Any]) + + +class timer(ContextDecorator): # noqa: N801 + """Simple timer for timing the execution of code. + + It can be used as either a context manager or a function decorator. The timing result will be logged upon exit. + + Example: + def func_a(): + time.sleep(1) + with timer("func_a"): + func_a() + + @timer("func_b) + def func_b(): + time.sleep(1) + func_b() + """ + + def __init__(self, context: str, debug: bool = False): + self.context = context + self.debug = debug + + def __enter__(self) -> None: + self.tic = time.time() + + def __exit__(self, exc_type, exc_value, traceback) -> None: # noqa: ANN001 + time_spent = time.time() - self.tic + if self.debug: + log.debug(f"Time spent on {self.context}: {time_spent:.4f} seconds") + else: + log.debug(f"Time spent on {self.context}: {time_spent:.4f} seconds") + + def __call__(self, func: T) -> T: + @functools.wraps(func) + def wrapper(*args, **kwargs): # noqa: ANN202 + tic = time.time() + result = func(*args, **kwargs) + time_spent = time.time() - tic + if self.debug: + log.debug(f"Time spent on {self.context}: {time_spent:.4f} seconds") + else: + log.debug(f"Time spent on {self.context}: {time_spent:.4f} seconds") + return result + + return wrapper # type: ignore + + +class Color: + """A convenience class to colorize strings in the console. + + Example: + import + print("This is {Color.red('important')}.") + """ + + @staticmethod + def red(x: str) -> str: + return termcolor.colored(str(x), color="red") + + @staticmethod + def green(x: str) -> str: + return termcolor.colored(str(x), color="green") + + @staticmethod + def cyan(x: str) -> str: + return termcolor.colored(str(x), color="cyan") + + @staticmethod + def yellow(x: str) -> str: + return termcolor.colored(str(x), color="yellow") diff --git a/nemo_vfm/physicalai/Dockerfile b/nemo_vfm/physicalai/Dockerfile new file mode 100644 index 00000000..baf798fb --- /dev/null +++ b/nemo_vfm/physicalai/Dockerfile @@ -0,0 +1,24 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvcr.io/nvidia/nemo:25.02.rc3 + +RUN pip install --no-cache-dir imageio[ffmpeg] pyav iopath better_profanity peft lru-dict pyquaternion git+https://github.com/NVlabs/Pytorch_Retinaface.git@b843f45 git+https://github.com/NVIDIA/NeMo-Run.git@f07877f80dbfc91c4d37683864cf2552b96d62f1 +RUN pip install moviepy==1.0.3 + +RUN rm -rf /opt/megatron-lm +RUN git clone --single-branch --branch cosmos1.0.1 https://github.com/NVIDIA/Megatron-LM.git /opt/megatron-lm && cd /opt/megatron-lm && git checkout 86dad2c39a52c241e372b774f07041201f8a57f4 && pip install -e . && cd megatron/core/datasets && make + +RUN rm -rf /opt/NeMo +COPY . /opt/NeMo \ No newline at end of file diff --git a/nemo_vfm/physicalai/__init__.py b/nemo_vfm/physicalai/__init__.py new file mode 100644 index 00000000..341a77c5 --- /dev/null +++ b/nemo_vfm/physicalai/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/datasets/__init__.py b/nemo_vfm/physicalai/datasets/__init__.py new file mode 100644 index 00000000..341a77c5 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/datasets/dataverse/.gitignore b/nemo_vfm/physicalai/datasets/dataverse/.gitignore new file mode 100644 index 00000000..fcfd00ca --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/.gitignore @@ -0,0 +1,168 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +outputs/** +.tmp/ +.vscode/ + +*.html +*.safetensors +*.ckpt +.idea +*.csv +*.py.swp +/configs/local_debug/* +.pypirc diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/__init__.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/__init__.py new file mode 100644 index 00000000..f77fbbe5 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from .utils.version import get_version + + +__version__ = get_version() diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/__init__.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/__init__.py new file mode 100644 index 00000000..0d4143e9 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/__init__.py @@ -0,0 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from enum import Enum + + +class DataField(Enum): + # [B, C, H, W], float32, RGB image ranges from 0 to 1. + IMAGE_RGB = "image_rgb" + # [B, 4, 4], float32, camera-to-world transformation matrix. + CAMERA_C2W_TRANSFORM = "camera_c2w_transform" + # [B, 4], float32, OpenCV pinhole intrinsics represented as [fx, fy, cx, cy]. + CAMERA_INTRINSICS = "camera_intrinsics" + # list of captions of size B. + CAPTION = "caption" + # [B, H, W], float32, depth map in metric scale. + METRIC_DEPTH = "metric_depth" + # [B, H, W], uint8, instance mask (0 is background). + DYNAMIC_INSTANCE_MASK = "dynamic_instance_mask" + # [B, H, W], float32, backward flow from this frame to previous frame. + BACKWARD_FLOW = "backward_flow" + # [B, H, W, 3], float32, ray direction (assume no motion/RS). + RAY_DIRECTION = "ray_direction" + # TODO [Add description] + OBJECT_BBOX = "object_bbox" + # TODO [Add description] a list of float32 point cloud. + POINT_CLOUD = "point_cloud" + # [B, N, (3 + 3x3)], N future positions. For the last dim, + # the first 3 are xyz locations, and tha last 9 are rots + # B corresponds to the number of timestamps for the base camera type + TRAJECTORY = "trajectory" diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/alpamayo_v1.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/alpamayo_v1.py new file mode 100644 index 00000000..6634affb --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/alpamayo_v1.py @@ -0,0 +1,391 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import json +import math +import os +import tarfile +import zlib +from pathlib import Path +from typing import Any, List + +import decord +import numpy as np +import torch +from lru import LRU +from platformdirs import user_cache_path +from pyquaternion import Quaternion +from scipy.spatial.transform import Rotation +from torch.nn.functional import grid_sample + +from dataverse.datasets import DataField +from dataverse.utils.ndas.av_metadata import ( + extract_calibration_from_tar, + extract_obstacle_and_epomotion_from_tar, + get_clip_to_tar, + get_egopose_interp, + get_obstacle_interp, + get_rig_transform, + key2id, + pose_to_corn, +) +from dataverse.utils.ndas.camera_model import FThetaCamera, IdealPinholeCamera + +from .base import BaseDataset + + +class MetadataLoaderV1(object): + def __init__( + self, + tar_dir=None, + obs_pose_dir=None, + calib_dir=None, + load_obstacle=False, + ): + self.tar_dir = tar_dir + self.obstaclepose_clip2tar = get_clip_to_tar(obs_pose_dir) + self.calibration_clip2tar = get_clip_to_tar(calib_dir) + self.load_obstacle = load_obstacle + + # Find common keys of the metadata + self.tar_index = self._build_tar_index() + annotated_clip_keys = set(self.obstaclepose_clip2tar.keys()).intersection( + set(self.calibration_clip2tar.keys()) + ) + + tar_clip_names = set(self.tar_index.keys()) + for clip_name in tar_clip_names: + if key2id(clip_name) not in annotated_clip_keys: + del self.tar_index[clip_name] + self.clip_names = list(self.tar_index.keys()) + + def _build_tar_index(self): + assert self.tar_dir is not None + tar_files = list(Path(self.tar_dir).glob("*.tar")) + tar_index_path = user_cache_path("dataverse") / "av_tar_index.json" + tar_index_path.parent.mkdir(parents=True, exist_ok=True) + + if tar_index_path.exists(): + with tar_index_path.open("r") as f: + tar_index = json.load(f) + + else: + tar_index = {} + for tar_file in tar_files: + with tarfile.open(tar_file, "r") as tar: + tar_members = tar.getmembers() + tar_members = [member.name.split(".")[0] for member in tar_members] + tar_members = list(set(tar_members)) + for clip_name in tar_members: + tar_index[clip_name] = str(tar_file) + with tar_index_path.open("w") as f: + json.dump(tar_index, f) + + if len(set(tar_index.values())) != len(tar_files): + print("Cache sanity check failed. Rebuilding") + os.unlink(tar_index_path) + return self._build_tar_index() + + return tar_index + + def load_metadata(self, sample_key, camkeys): + """Returns dictionary for the clip with the following contents: + { + camera_key_i: + { + sensor_name: str, name of the lidar sensor + egopose: list, egoposes sorted in time + lidar2rig: array, lidar to rig transformation + rig2cam: array, rig to camera transformation + ftheta: Ftheta camera object + obstacles: dict, contains trackid->[detection1,detection2,...] sorted in time + } + } + """ + + # Load obstacles, egoposes, sensor_name + metadata = extract_obstacle_and_epomotion_from_tar( + sample_key, + camkeys, + self.obstaclepose_clip2tar, + load_obstacle=self.load_obstacle, + ) + if not metadata: + return None + + # Load rig_info egoparams + calibration = extract_calibration_from_tar(sample_key, self.calibration_clip2tar) + if not calibration: + return None + + # Merge calibration to metadata + rig_info = calibration["rig_info"] + metadata["egoparams"] = calibration["egoparams"] + for camera_name in camkeys: + camera_data = metadata[camera_name] + lidarkey = camera_data["sensor_name"] + camera_data["lidar2rig"] = get_rig_transform(rig_info[lidarkey], rig2sensor=False) + camera_data["rig2cam"] = get_rig_transform(rig_info[camera_name + ".mp4"], rig2sensor=True) + camera_data["ftheta"] = FThetaCamera.from_dict(rig_info[camera_name + ".mp4"]) + return metadata + + +class AlpamayoV1(BaseDataset): + MAX_CACHED_VIDEOS = 2 + ROT_FIXER = torch.from_numpy(Rotation.from_euler("xzy", [0.0, -np.pi / 2, np.pi / 2]).as_matrix())[None].float() + + def __init__( + self, + tar_dir: str, + calib_dir: str, + obs_pose_dir: str, + frame_height: int = 720, + frame_width: int = 1280, + fov_range: list = [30, 70], + fov_rng_seed: int = 0, + multi_cam: bool = False, + ): + super().__init__() + + if multi_cam: + self.camkeys = [ + "camera_front_wide_120fov", + "camera_cross_left_120fov", + "camera_cross_right_120fov", + ] + else: + self.camkeys = ["camera_front_wide_120fov"] + self.metadata_loader = MetadataLoaderV1( + tar_dir=tar_dir, + obs_pose_dir=obs_pose_dir, + calib_dir=calib_dir, + ) + + self.tar_dir = tar_dir + self.frame_height = frame_height + self.frame_width = frame_width + self.fov_range = fov_range + self.fov_rng_seed = fov_rng_seed + self.video_loader_cache = LRU(self.MAX_CACHED_VIDEOS) + self.meta_data_cache = LRU(self.MAX_CACHED_VIDEOS) + + def num_videos(self) -> int: + return len(self.metadata_loader.clip_names) + + def num_views(self, video_idx: int) -> int: + return len(self.camkeys) + + def num_frames(self, video_idx: int, view_idx: int = 0) -> int: + loaders, _ = self._get_video_loader(video_idx) + return len(loaders[self.camkeys[view_idx]]) + + def available_data_fields(self) -> List[DataField]: + return [ + DataField.IMAGE_RGB, + DataField.CAMERA_C2W_TRANSFORM, + DataField.CAMERA_INTRINSICS, + ] + + def _get_video_loader(self, video_idx: int): + if video_idx in self.video_loader_cache: + return self.video_loader_cache[video_idx] + + clip_name = self.metadata_loader.clip_names[video_idx] + video_loaders, video_info = {}, {} + tar_name = self.metadata_loader.tar_index[clip_name] + + with tarfile.open(os.path.join(self.tar_dir, tar_name), "r") as f: + for key in self.camkeys: + mp4_name = f"{clip_name}.{key}.mp4" + + vr = decord.VideoReader(f.extractfile(mp4_name)) + video_loaders[key] = vr + + infokey = mp4_name.replace(".mp4", ".json") + vidinfo = json.load(f.extractfile(infokey)) + video_info[key] = vidinfo + + return video_loaders, video_info + + def _get_meta_data(self, video_idx: int): + if video_idx in self.meta_data_cache: + return self.meta_data_cache[video_idx] + + clip_name = self.metadata_loader.clip_names[video_idx] + mdata = self.metadata_loader.load_metadata(clip_name, self.camkeys) + + ego_data = {} + for camera_name in self.camkeys: + camera_metadata = mdata[camera_name] + egopose_lerp, cliptbase = get_egopose_interp(camera_metadata["egopose"]) + if "obstacles" in camera_metadata: + obstacle_lerp = get_obstacle_interp(["obstacles"], egopose_lerp, cliptbase) + else: + obstacle_lerp = None + ego_data[camera_name] = { + "egopose_lerp": egopose_lerp, + "cliptbase": cliptbase, + "obstacle_lerp": obstacle_lerp, + } + + return mdata, ego_data + + @staticmethod + def _get_obstacles_and_camera( + egopose_lerp, + lidar2rig, + rig2cam, + t0, + obstacle_lerp=None, + ): + # Process camera poses + # convert global lidar -> local lidar + ego_qw, ego_qx, ego_qy, ego_qz, ego_x, ego_y, ego_z = egopose_lerp["interp"](t0) + ego_quat = Quaternion(w=ego_qw, x=ego_qx, y=ego_qy, z=ego_qz) + + # convert global lidar -> local lidar + global2lidar_trans = np.eye(4) + global2lidar_trans[:3, 3] = -np.array([ego_x, ego_y, ego_z]) + global2lidar_rot = np.eye(4) + global2lidar_rot[:3, :3] = ego_quat.inverse.rotation_matrix + + # calculate world to camera transform + # Lidar Global -> Lidar Local -> Rig -> Camera + world2cam = rig2cam @ lidar2rig @ global2lidar_rot @ global2lidar_trans + + # Process obstacles + pts = np.empty((0, 3)) + if obstacle_lerp: + # obstacle points (8 corners) + N = 0 + for id in obstacle_lerp: + if obstacle_lerp[id]["tmin"] <= t0 <= obstacle_lerp[id]["tmax"]: + qw, qx, qy, qz, x, y, z, le, wi, he = obstacle_lerp[id]["interp"](t0) + corn = pose_to_corn(qw, qx, qy, qz, x, y, z, le, wi, he) + pts = np.concatenate((pts, corn), 0) + N += 1 + + pts = pts.astype(np.float32) + + # append 1 to pts + pts = np.concatenate((pts, np.ones((pts.shape[0], 1))), axis=1) + + pts = pts @ world2cam.T + pts = pts[:, :3] + + # pts = pts[pts[..., 0] > 0] + + return pts, world2cam + + def _read_data( + self, + video_idx: int, + frame_idxs: List[int], + view_idxs: List[int], + data_fields: List[DataField], + ) -> dict[DataField, Any]: + # Load videos + clip_name = self.metadata_loader.clip_names[video_idx] + video_loaders, video_info = self._get_video_loader(video_idx) + meta_data, ego_data = self._get_meta_data(video_idx) + + # Set fov if randomized + seed = zlib.adler32(f"{clip_name}-{self.fov_rng_seed}".encode("utf-8")) + pinhole_fovd = np.random.RandomState(seed).randint(self.fov_range[0], self.fov_range[1] + 1) + # target pinhole image/camera in the original resolution + tgt_focal = self.frame_width / (2.0 * math.tan(np.deg2rad(pinhole_fovd) / 2.0)) + tgt_focal = np.array([tgt_focal, tgt_focal]) + ys, xs = np.mgrid[0 : self.frame_height, 0 : self.frame_width] + pixels = np.stack((xs, ys), axis=2) + pinhole_cam = IdealPinholeCamera( + f_x=tgt_focal[0], + f_y=tgt_focal[1], + width=self.frame_width, + height=self.frame_height, + ) + pinhole_rays = pinhole_cam.pixel2ray(pixels.reshape(-1, 2)) # hw x 3 + + # Pre-compute projection for all possible cameras. + pos_norms = {} + for view_idx in set(view_idxs): + cam_key = self.camkeys[view_idx] + fisheye_cam = meta_data[cam_key]["ftheta"] + pos = fisheye_cam.ray2pixel(pinhole_rays) + # fish eye to pinhole + pos_norm = ( + 2.0 + * pos + / np.array( + [fisheye_cam.width - 1.0, fisheye_cam.height - 1.0], + dtype=np.float32, + ) + - 1.0 + ) + pos_norm = torch.from_numpy(pos_norm).reshape(1, self.frame_height, self.frame_width, 2) + pos_norms[view_idx] = pos_norm + + output_dict = {} + for data_field in data_fields: + if data_field == DataField.IMAGE_RGB: + rgb_list = [] + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + cam_key = self.camkeys[view_idx] + pos_norm = pos_norms[view_idx] + img_fisheye = video_loaders[cam_key][frame_idx].asnumpy() + img = torch.from_numpy(img_fisheye).permute(2, 0, 1).float().unsqueeze(0) + img = grid_sample(img, pos_norm, mode="bilinear", align_corners=False)[0] + img = img.float() / 255.0 + rgb_list.append(img) + + output_dict[data_field] = torch.stack(rgb_list) + + elif data_field == DataField.CAMERA_C2W_TRANSFORM: + c2w_list = [] + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + cam_key = self.camkeys[view_idx] + t0_stamp = video_info[cam_key][frame_idx]["timestamp"] + t0_val = 1e-6 * (t0_stamp - ego_data[cam_key]["cliptbase"]) + + _, world2cam = self._get_obstacles_and_camera( + ego_data[cam_key]["egopose_lerp"], + meta_data[cam_key]["lidar2rig"], + meta_data[cam_key]["rig2cam"], + t0_val, + obstacle_lerp=None, + ) + + world2cam = torch.from_numpy(world2cam).float() + cam2world = torch.inverse(world2cam) + cam2world[:3, :3] = cam2world[:3, :3] @ self.ROT_FIXER + c2w_list.append(cam2world) + + output_dict[data_field] = torch.stack(c2w_list) + + elif data_field == DataField.CAMERA_INTRINSICS: + single_intr = torch.tensor( + [ + tgt_focal[0], + tgt_focal[1], + self.frame_width / 2, + self.frame_height / 2, + ] + ).float() + output_dict[data_field] = single_intr.repeat(len(frame_idxs), 1) + + else: + raise NotImplementedError(f"Can't handle data field {data_field}") + + return output_dict diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/alpamayo_v2.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/alpamayo_v2.py new file mode 100644 index 00000000..d0ba4fd8 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/alpamayo_v2.py @@ -0,0 +1,491 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import io +import json +import math +import os +import random +import tarfile +import zlib +from pathlib import Path +from typing import Any, List + +import av +import numpy as np +import torch +from lru import LRU +from omegaconf import DictConfig +from platformdirs import user_cache_path +from pyquaternion import Quaternion +from scipy.spatial.transform import Rotation +from torch.nn.functional import grid_sample + +import dataverse.utils.alpamayo.egomotion_decoder as egomotion_decoder +from dataverse.utils.ndas.av_metadata import get_egopose_interp, get_rig_transform, parse_calibration_data +from dataverse.utils.ndas.camera_model import FThetaCamera, IdealPinholeCamera + +from .base import BaseDataset, DataField + + +class AlpamayoV2(BaseDataset): + MAX_CACHED_VIDEOS = 2 + ROT_FIXER = torch.from_numpy(Rotation.from_euler("xzy", [0.0, -np.pi / 2, np.pi / 2]).as_matrix())[None].float() + + def __init__( + self, + tar_dirs: list[str], + probe_tar: bool, + rectify: DictConfig, + camkeys: list[str] = ["camera_front_wide_120fov"], + tar_cache_path: str = "av22_tar_index_full.json", + ): + super().__init__() + + self.camkeys = camkeys + self.trajectory_base_camera_index = self.camkeys.index("camera_front_wide_120fov") + self.tar_cache_path = tar_cache_path + self.tar_dirs = tar_dirs + self.rectify_cfg = rectify + self.probe_tar = probe_tar + + # Mapping from clip_name to tar file + self.tar_index = self._build_tar_index() + self.clip_names = sorted(list(self.tar_index.keys())) + + self.video_loader_cache = LRU(self.MAX_CACHED_VIDEOS) + self.meta_data_cache = LRU(self.MAX_CACHED_VIDEOS) + + def _build_tar_index(self): + """ + Builds the tar index from tar files, using a cached version if available. + Ensures consistency in cache and rebuilds if the cache sanity check fails. + + Returns: + dict: A mapping of clip names to their respective tar file paths. + """ + tar_files = [] + for tar_dir in self.tar_dirs: + tar_files.extend([str(p) for p in Path(tar_dir).glob("*.tar")]) + tar_index_path = user_cache_path("dataverse") / self.tar_cache_path + tar_index_path.parent.mkdir(parents=True, exist_ok=True) + + if tar_index_path.exists(): + with tar_index_path.open("r") as f: + tar_index = json.load(f) + if len(set(tar_index.values())) != len(tar_files): + print("Cache sanity check failed. Rebuilding") + os.unlink(tar_index_path) + return self._build_tar_index() + else: + tar_index = {} + for tar_file in tar_files: + if self.probe_tar: + clip_name = Path(tar_file).stem + tar_index[clip_name] = str(tar_file) + continue + with tarfile.open(tar_file, "r") as tar: + tar_members = tar.getmembers() + tar_members = [member.name.split(".")[0] for member in tar_members] + tar_members = list(set(tar_members)) + for clip_name in tar_members: + tar_index[clip_name] = str(tar_file) + with tar_index_path.open("w") as f: + json.dump(tar_index, f) + + return tar_index + + def num_videos(self) -> int: + return len(self.clip_names) + + def num_views(self, video_idx: int) -> int: + return len(self.camkeys) + + def num_frames(self, video_idx: int, view_idx: int = 0) -> int: + _, _, _, sync_to_original, _ = self._get_meta_data(video_idx) + return len(sync_to_original[self.camkeys[view_idx]]) + + def available_data_fields(self) -> List[DataField]: + fields = [ + DataField.IMAGE_RGB, + DataField.CAMERA_C2W_TRANSFORM, + DataField.RAY_DIRECTION, + DataField.TRAJECTORY, + ] + if self.rectify_cfg.enabled: + fields.append(DataField.CAMERA_INTRINSICS) + return fields + + def _get_video_frames( + self, + video_idx: int, + view_idxs: list[int], + frame_idxs: list[int], + sync_to_original: dict, + ): + clip_name = self.clip_names[video_idx] + tar_path = self.tar_index[clip_name] + + video_frames = {} + frame_idx_to_batch_idx = {} + with tarfile.open(tar_path, "r") as f: + for key in self.camkeys: + # get frame_idxs of the current view + target_frame_idxs = [] + for view_idx, frame_idx in zip(view_idxs, frame_idxs): + if self.camkeys[view_idx] == key: + target_frame_idxs.append(frame_idx) + + if len(target_frame_idxs) == 0: + # this camera was not requested in view_idxs + break + + # check if frame_idx are in an increasing order + diff = np.diff(np.array(target_frame_idxs)) + assert np.all(diff > 0), "frame_idxs are given as non-increasing list" + + # read video of the current view + fp = f.extractfile(f"{clip_name}.{key}.mp4") + assert fp is not None, f"Video {clip_name}.{key}.mp4 not found" + vid = io.BytesIO(fp.read()) + vid.seek(0) + + input_container = av.open(vid) + input_container.streams.video[0].thread_type = 3 + average_fps = input_container.streams.video[0].average_rate + time_base = input_container.streams.video[0].time_base + average_frame_duration = int(1 / average_fps / time_base) + + # collect required frames of the current view + frames = [] + frame_iterator = input_container.decode(video=0) + cur_frame_idx_to_batch_idx = {} + for batch_idx, target_frame_number in enumerate(target_frame_idxs): + adjusted_target_frame_number = sync_to_original[key][target_frame_number] + target_pts = adjusted_target_frame_number * average_frame_duration + for frame in frame_iterator: + # find the frame + if frame.pts == target_pts: + break + + cur_frame_idx_to_batch_idx[target_frame_number] = batch_idx + frames.append(torch.as_tensor(frame.to_rgb().to_ndarray())) + video_frames[key] = torch.stack(frames) + frame_idx_to_batch_idx[key] = cur_frame_idx_to_batch_idx + + return video_frames, frame_idx_to_batch_idx + + @staticmethod + def _extract_egomotion_from_sample(fp): + """Extract egomotions from sample, used by AV2.1""" + egopose_info = np.load(fp, allow_pickle=True) + N, C = egopose_info["labels_data"].shape + (K,) = egopose_info["labels_keys"].shape + assert K == C, f"{K} {C}" + + egopose_parsed = [{} for i in range(N)] + sensor_names = [] + target_keys = ["x", "y", "z", "qx", "qy", "qz", "qw", "timestamp"] + for key_id, key in enumerate(egopose_info["labels_keys"]): + values = egopose_info["labels_data"][:, key_id] + if key == "coordinate_frame": + sensor_names = values + continue + elif key not in target_keys: + continue + if key == "timestamp": + key = "t" + for i in range(N): + egopose_parsed[i][key] = values[i] + return { + "egopose": egopose_parsed, + "sensor_name": sensor_names[0], + } + + @staticmethod + def _extract_calibration_from_sample(fp): + """Extract calibrations from sample, used by AV2.1""" + calibration = json.load(fp) + rig_info, egoparams = parse_calibration_data(calibration) + return {"rig_info": rig_info, "egoparams": egoparams, "rig_raw": calibration} + + def _get_meta_data(self, video_idx: int): + """Returns dictionary for the clip with the following contents: + { + camera_key_i: + { + sensor_name: str, name of the lidar sensor + egopose: list, egoposes sorted in time + lidar2rig: array, lidar to rig transformation + rig2cam: array, rig to camera transformation + ftheta: Ftheta camera object + } + } + """ + + if video_idx in self.meta_data_cache: + return self.meta_data_cache[video_idx] + + clip_name = self.clip_names[video_idx] + tar_path = self.tar_index[clip_name] + video_info = {} + + with tarfile.open(tar_path, "r") as f: + camera_metadata = self._extract_egomotion_from_sample(f.extractfile(f"{clip_name}.egomotion.npz")) + calibration = self._extract_calibration_from_sample(f.extractfile(f"{clip_name}.rig.json")) + + for key in self.camkeys: + vidinfo = json.load(f.extractfile(f"{clip_name}.{key}.json")) + video_info[key] = vidinfo + + # Synchronize timestamps + min_t, max_t = np.inf, -np.inf + original_timestamps = {} + for key in self.camkeys: + timestamps = np.array([info["timestamp"] for info in video_info[key]]) + min_t = min(min_t, timestamps.min()) + max_t = max(max_t, timestamps.max()) + original_timestamps[key] = timestamps + ref_timestamps = original_timestamps[self.camkeys[0]] + ref_timestamps = ref_timestamps[ref_timestamps >= min_t] + ref_timestamps = ref_timestamps[ref_timestamps <= max_t] + + sync_to_original = {} + for key in self.camkeys: + sync_to_original[key] = np.searchsorted(original_timestamps[key], ref_timestamps) + + metadata, ego_data = {}, {} + lidarkey = camera_metadata["sensor_name"] + rig_info = calibration["rig_info"] + metadata["egoparams"] = calibration["egoparams"] + for camera_name in self.camkeys: + camera_data = {} + camera_data["egopose"] = camera_metadata["egopose"] + camera_data["sensor_name"] = lidarkey + camera_data["lidar2rig"] = get_rig_transform(rig_info[lidarkey], rig2sensor=False) + camera_data["rig2cam"] = get_rig_transform(rig_info[camera_name + ".mp4"], rig2sensor=True) + camera_data["ftheta"] = FThetaCamera.from_dict(rig_info[camera_name + ".mp4"]) + egopose_lerp, cliptbase = get_egopose_interp(camera_data["egopose"]) + + metadata[camera_name] = camera_data + ego_data[camera_name] = { + "egopose_lerp": egopose_lerp, + "cliptbase": cliptbase, + } + + return metadata, ego_data, video_info, sync_to_original, calibration["rig_raw"] + + @staticmethod + def _get_obstacles_and_camera(egopose_lerp, lidar2rig, rig2cam, t0): + # Process camera poses + # convert global lidar -> local lidar + ego_qw, ego_qx, ego_qy, ego_qz, ego_x, ego_y, ego_z = egopose_lerp["interp"](t0) + ego_quat = Quaternion(w=ego_qw, x=ego_qx, y=ego_qy, z=ego_qz) + + # convert global lidar -> local lidar + global2lidar_trans = np.eye(4) + global2lidar_trans[:3, 3] = -np.array([ego_x, ego_y, ego_z]) + global2lidar_rot = np.eye(4) + global2lidar_rot[:3, :3] = ego_quat.inverse.rotation_matrix + + # calculate world to camera transform + # Lidar Global -> Lidar Local -> Rig -> Camera + world2cam = rig2cam @ lidar2rig @ global2lidar_rot @ global2lidar_trans + + return world2cam + + def _get_rectify_pinhole_info(self, video_idx: int, view_idxs: List[int]): + cfg = self.rectify_cfg + + # Set fov if randomized + seed = zlib.adler32(f"{self.clip_names[video_idx]}-{cfg.fov_rng_seed}".encode("utf-8")) + pinhole_fovd = np.random.RandomState(seed).randint(cfg.fov_range[0], cfg.fov_range[1] + 1) + # target pinhole image/camera in the original resolution + tgt_focal = cfg.width / (2.0 * math.tan(np.deg2rad(pinhole_fovd) / 2.0)) + tgt_focal = np.array([tgt_focal, tgt_focal]) + ys, xs = np.mgrid[0 : cfg.height, 0 : cfg.width] + pixels = np.stack((xs, ys), axis=2) + pinhole_cam = IdealPinholeCamera( + f_x=tgt_focal[0], + f_y=tgt_focal[1], + width=cfg.width, + height=cfg.height, + ) + pinhole_intr = torch.tensor( + [ + tgt_focal[0], + tgt_focal[1], + cfg.width / 2, + cfg.height / 2, + ] + ).float() + pinhole_rays = pinhole_cam.pixel2ray(pixels.reshape(-1, 2)) # hw x 3 + + # Pre-compute projection for all possible cameras. + meta_data, _, _, _, _ = self._get_meta_data(video_idx) + + pos_norms = {} + for view_idx in set(view_idxs): + cam_key = self.camkeys[view_idx] + fisheye_cam = meta_data[cam_key]["ftheta"] + pos = fisheye_cam.ray2pixel(pinhole_rays) + # fish eye to pinhole + pos_norm = ( + 2.0 + * pos + / np.array( + [fisheye_cam.width - 1.0, fisheye_cam.height - 1.0], + dtype=np.float32, + ) + - 1.0 + ) + pos_norm = torch.from_numpy(pos_norm).reshape(1, cfg.height, cfg.width, 2) + pos_norms[view_idx] = pos_norm + + return pinhole_rays, pinhole_intr, pos_norms + + def egomotion_alpamayo_parser(self, video_idx, timestamps): + clip_name = self.clip_names[video_idx] + tar_path = self.tar_index[clip_name] + + ed_config = egomotion_decoder.EgoMotionDecoderConfig(decode_strategy=f"uniform_{len(timestamps)}_frame") + with tarfile.open(tar_path, "r") as f: + data = {} + data["egomotion.npz"] = f.extractfile(f"{clip_name}.egomotion.npz").read() + data["live_egomotion.npz"] = f.extractfile(f"{clip_name}.live_egomotion.npz").read() + data["rig.json"] = f.extractfile(f"{clip_name}.rig.json").read() + + ego_data = egomotion_decoder.decode_egomotion(data, timestamps, ed_config) + + # flatten xyz and rotation into a 12-dim vector + xyz = ego_data["ego_future_xyz"] + rot = ego_data["ego_future_rot"] + # traj is of shape (B, N, 12) where N is the number of futuer ego poses + # default N = 64, 6.4 seconds, each one sampled at 0.1 sec interval. + traj = torch.cat([xyz, rot.flatten(2, 3)], dim=-1) + return traj + + def _read_data( + self, + video_idx: int, + frame_idxs: List[int], + view_idxs: List[int], + data_fields: List[DataField], + ) -> dict[DataField, Any]: + meta_data, ego_data, video_info, sync_to_original, rig_info = self._get_meta_data(video_idx) + video_frames, frame_idx_to_batch_idx = self._get_video_frames( + video_idx, view_idxs, frame_idxs, sync_to_original + ) + + if self.rectify_cfg.enabled: + cam_rays, tgt_intr, pos_norms = self._get_rectify_pinhole_info(video_idx, view_idxs) + rays = {view_idx: cam_rays for view_idx in set(view_idxs)} + else: + tgt_intr, pos_norms = None, None + rays = {} + for view_idx in set(view_idxs): + fisheye_cam: FThetaCamera = meta_data[self.camkeys[view_idx]]["ftheta"] + height, width = video_frames[self.camkeys[view_idx]][0].shape[:2] + ys, xs = np.meshgrid( + np.linspace(0, fisheye_cam.height - 1, height), + np.linspace(0, fisheye_cam.width - 1, width), + indexing="ij", + ) + pixels = np.stack((xs, ys), axis=2).reshape(-1, 2) + cam_rays = fisheye_cam.pixel2ray(pixels)[0].reshape(height, width, 3) + rays[view_idx] = cam_rays + + output_dict = {} + for data_field in data_fields: + if data_field == DataField.IMAGE_RGB: + rgb_list = [] + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + cam_key = self.camkeys[view_idx] + batch_idx = frame_idx_to_batch_idx[cam_key][frame_idx] + img_fisheye = video_frames[cam_key][batch_idx] + + img = img_fisheye.permute(2, 0, 1).float().unsqueeze(0) + if pos_norms is not None: + img = grid_sample( + img, + pos_norms[view_idx], + mode="bilinear", + align_corners=False, + ) + img = img[0].float() / 255.0 + rgb_list.append(img) + output_dict[data_field] = torch.stack(rgb_list) + + elif data_field == DataField.CAMERA_C2W_TRANSFORM: + c2w_list = [] + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + cam_key = self.camkeys[view_idx] + t0_stamp = video_info[cam_key][sync_to_original[cam_key][frame_idx]]["timestamp"] + t0_val = 1e-6 * (t0_stamp - ego_data[cam_key]["cliptbase"]) + + world2cam = self._get_obstacles_and_camera( + ego_data[cam_key]["egopose_lerp"], + meta_data[cam_key]["lidar2rig"], + meta_data[cam_key]["rig2cam"], + t0_val, + ) + + world2cam = torch.from_numpy(world2cam).float() + cam2world = torch.inverse(world2cam) + cam2world[:3, :3] = cam2world[:3, :3] @ self.ROT_FIXER + c2w_list.append(cam2world) + + output_dict[data_field] = torch.stack(c2w_list) + + elif data_field == DataField.CAMERA_INTRINSICS: + assert tgt_intr is not None + output_dict[data_field] = tgt_intr.repeat(len(frame_idxs), 1) + + elif data_field == DataField.RAY_DIRECTION: + ray_list = [] + for view_idx in view_idxs: + ray_list.append(torch.from_numpy(rays[view_idx]).float()) + output_dict[data_field] = torch.stack(ray_list) + elif data_field == DataField.TRAJECTORY: + unique_view_idxs = list(set(view_idxs)) + view_count = view_idxs.count(unique_view_idxs[0]) + for view_idx in unique_view_idxs[1:]: + assert view_count == view_idxs.count(view_idx), ( + "trajectory datafield expects equal number of frames per view" + ) + + # choose one camera timestamps to compute trajectory + if self.trajectory_base_camera_index in view_idxs: + target_traj_cam_index = self.trajectory_base_camera_index + else: + target_traj_cam_index = random.sample(view_idxs, 1)[0] + timestamps = [] + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + if view_idx == target_traj_cam_index: + # compute timestamps of the target base camera + cam_key = self.camkeys[view_idx] + t0_stamp = video_info[cam_key][sync_to_original[cam_key][frame_idx]]["timestamp"] + timestamps.append(t0_stamp) + if len(timestamps) == 0: + raise ValueError("view_idxs do not contain the base camera for trajectory computation.") + + # compute trajectory for the target base camera + output_dict[data_field] = self.egomotion_alpamayo_parser(video_idx, timestamps) + output_dict["rig_info"] = rig_info + else: + raise NotImplementedError(f"Can't handle data field {data_field}") + + return output_dict diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/base.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/base.py new file mode 100644 index 00000000..324399ad --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/base.py @@ -0,0 +1,134 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Any, List, Optional + +from . import DataField + + +class BaseDataset(ABC): + """ + Base class for all datasets. + Note that this is not directly wrap-able by a dataloader. It is meant to be + subclassed / included in another dataset class. + """ + + def __init__(self): + pass + + @abstractmethod + def available_data_fields(self) -> List[DataField]: + """ + Return a list of available data fields in the dataset. + """ + pass + + @abstractmethod + def num_videos(self) -> int: + """ + Returns: + Number of videos in the dataset. + """ + pass + + @abstractmethod + def num_views(self, video_idx: int) -> int: + """ + Args: + video_idx: Index of the video. + + Returns: + Number of views in the video. + """ + pass + + @abstractmethod + def num_frames(self, video_idx: int, view_idx: int = 0) -> int: + """ + Args: + video_idx: Index of the video. + view_idx: Index of the view. + + Returns: + Number of frames in the given view. + """ + pass + + def read( + self, + video_idx: int, + frame_idxs: Optional[List[int]] = None, + view_idxs: Optional[List[int]] = None, + data_fields: Optional[List[DataField]] = None, + ) -> dict[DataField, Any]: + """ + Read data from the dataset. + Args: + video_idx: Index of the video. + frame_idxs: List of frame indices. + view_idxs: List of view indices. + data_fields: List of data fields to read. If None, read all data fields. + + Example: + if frame_idxs is None, view_idxs is None, read all frames from all views. + if frame_idxs is not None, view_idxs is None, read frames from the first view. + if frame_idxs is None, view_idxs is not None, read all frames from the specified views. + if frame_idxs is not None, view_idxs is not None, read frames from the specified views. + + Returns: + A dictionary mapping data fields to their values. + """ + + if data_fields is None: + data_fields = self.available_data_fields() + + if frame_idxs is None: + # Frame not provided, default read all frames. + if view_idxs is None: + view_iterator = range(self.num_views(video_idx)) + else: + view_iterator = view_idxs + + new_frame_idxs, new_view_idxs = [], [] + for view_idx in view_iterator: + num_frames = self.num_frames(video_idx, view_idx) + new_frame_idxs.extend(list(range(num_frames))) + new_view_idxs.extend([view_idx] * num_frames) + frame_idxs, view_idxs = new_frame_idxs, new_view_idxs + + elif view_idxs is None: + # View not provided, but frame is provided, we only read the first view. + view_idxs = [0] * len(frame_idxs) + + else: + # Both frame_idxs and view_idxs provided, do sanity check. + assert len(frame_idxs) == len(view_idxs), "Frame and view indices must match." + + return self._read_data( + video_idx=video_idx, + frame_idxs=frame_idxs, + view_idxs=view_idxs, + data_fields=data_fields, + ) + + @abstractmethod + def _read_data( + self, + video_idx: int, + frame_idxs: List[int], + view_idxs: List[int], + data_fields: List[DataField], + ) -> dict[DataField, Any]: + pass diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/cosmos_av_w_traj.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/cosmos_av_w_traj.py new file mode 100644 index 00000000..47169f69 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/datasets/cosmos_av_w_traj.py @@ -0,0 +1,475 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import glob +import io +import json +import os +import pickle +import random +import shutil +import tarfile +import tempfile +import time +import uuid +from typing import Any, List + +import av +import numpy as np +import pandas as pd +import torch +import tqdm +from einops import rearrange +from lru import LRU +from omegaconf import DictConfig +from platformdirs import user_cache_path + +import dataverse.utils.alpamayo.egomotion_decoder as egomotion_decoder + +from .alpamayo_v2 import AlpamayoV2 +from .base import DataField + + +class CosmosAV(AlpamayoV2): + def __init__( + self, + tar_dirs: list[str], + uuid_dirs: str, + t5_dirs: str, + probe_tar: bool, + rectify: DictConfig, + camkeys: list[str] = ["camera_front_wide_120fov"], + decode_traj: bool = False, + use_hq_data: bool = False, + use_non_mb_data: bool = False, + tar_cache_path: str = "av22_tar_index_full.json", + t5_cache_path: str = "av22_qwen_t5_index_full.json", + ): + super().__init__(tar_dirs, probe_tar, rectify, camkeys, tar_cache_path) + self.camkeys = camkeys + self.trajectory_base_camera_index = self.camkeys.index("camera_front_wide_120fov") + self.tar_dirs = tar_dirs + self.uuid_dirs = uuid_dirs + self.t5_dirs = t5_dirs + self.rectify_cfg = rectify + self.probe_tar = probe_tar + self.decode_traj = decode_traj + self.tar_cache_path = tar_cache_path + self.t5_cache_path = t5_cache_path + if use_hq_data: + self.t5_index = self._buiild_qwen_t5_dirs() + else: + self.t5_index = self._buiild_t5_dirs() # build clip name based on t5 index + clip_names = sorted(list(self.t5_index.keys())) + self.video_loader_cache = LRU(self.MAX_CACHED_VIDEOS) + self.meta_data_cache = LRU(self.MAX_CACHED_VIDEOS) + # Mapping from clip_name to tar file, remove the missing index + self.tar_index = self._build_tar_index() + missing_keys = [key for key in clip_names if key not in self.tar_index] + self.clip_names = [key for key in clip_names if key in self.tar_index] + print("Missing Clip number: {}".format(len(missing_keys))) + print(f"Total number of clips: {len(self.clip_names)}") + # Filter out unwanted data + if use_non_mb_data: + non_mb_clip = self._build_non_mb_data_idx() + if use_non_mb_data: + self.clip_names = [key for key in self.clip_names if key in non_mb_clip] + print(f"Totoal number of clips after filtering: {len(self.clip_names)}") + + def _build_non_mb_data_idx(self): + """ + Builds an index of non-MB clip IDs using a cached version if available. + + Returns: + set: A set of clip IDs that are not associated with the "mb" partner. + """ + clip_id_filtering_path = user_cache_path("dataverse") / self.tar_cache_path.replace(".json", "_non_mb.json") + clip_id_filtering_path.parent.mkdir(parents=True, exist_ok=True) + + if clip_id_filtering_path.exists(): + with clip_id_filtering_path.open("r") as f: + keep = json.load(f) + else: + clip_id_filtering_parquet_path = user_cache_path("dataverse") / "query_clip_id_filtering.parquet" + df = pd.read_parquet(clip_id_filtering_parquet_path) + keep = set() + for i, row in df.iterrows(): + if row.partner == "mb": + continue + keep.add(row["id"]) + + with clip_id_filtering_path.open("w") as f: + json.dump(list(keep), f) + return set(keep) + + def _buiild_t5_dirs(self): + """ + Builds the t5 index from cached mapping files. + Returns: + dict: The built t5 index dictionary. + """ + t5_index_path = user_cache_path("dataverse") / self.t5_cache_path + t5_index_path.parent.mkdir(parents=True, exist_ok=True) + if t5_index_path.exists(): + with t5_index_path.open("r") as f: + t5_index = json.load(f) + else: + t5_index = dict() + + for file in tqdm.tqdm(glob.glob(self.uuid_dirs + "*.parquet", recursive=True)): + cache_file_path = os.path.join( + file.replace(".parquet", "").replace(self.uuid_dirs, self.t5_dirs), + f"{file.replace('.parquet', '').split('/')[-1]}_mapping.json", + ) + + if os.path.exists(cache_file_path): + with open(cache_file_path, "r") as f: + curr_t5_index = json.load(f) + for key, value in curr_t5_index.items(): + clip_id, camera_id = key.split(".") + + if os.path.exists(value.replace(".tar", ".json")): + if clip_id not in t5_index: + t5_index[clip_id] = [] + t5_index[clip_id].append(value.split(self.t5_dirs)[-1]) + else: + print(f"{cache_file_path} missing") + + for key, value in t5_index.items(): + t5_index[key] = list(set(value)) + with t5_index_path.open("w") as f: + json.dump(t5_index, f) + print(f"**** Cache t5_index to {t5_index_path}") + return t5_index + + def _buiild_qwen_t5_dirs(self): + """ + Builds the t5 index using qwen from cached mapping files. + Returns: + dict: The built t5 index dictionary. + """ + t5_index_path = user_cache_path("dataverse") / self.t5_cache_path + t5_index_path.parent.mkdir(parents=True, exist_ok=True) + if t5_index_path.exists(): + with t5_index_path.open("r") as f: + t5_index = json.load(f) + else: + t5_index = dict() + for chunk_id in tqdm.tqdm(os.listdir(self.t5_dirs)): + cache_file_path = os.path.join(self.t5_dirs, chunk_id, "mapping.json") + if os.path.exists(cache_file_path): + with open(cache_file_path, "r") as f: + curr_t5_index = json.load(f) + for key, value in curr_t5_index.items(): + clip_id, camera_id = key.split(".") + + if os.path.exists(value.replace(".tar", ".json")): + if clip_id not in t5_index: + t5_index[clip_id] = [] + t5_index[clip_id].append(value.split(self.t5_dirs)[-1]) + else: + print(f"{cache_file_path} missing") + + for key, value in t5_index.items(): + t5_index[key] = list(set(value)) + with t5_index_path.open("w") as f: + json.dump(t5_index, f) + print(f"**** Cache t5_index to {t5_index_path}") + + return t5_index + + def _get_video_frames( + self, + video_idx: int, + view_idxs: list[int], + frame_idxs: list[int], + sync_to_original: dict, + ): + # rank = distributed.get_rank() + + clip_name = self.clip_names[video_idx] + tar_path = self.tar_index[clip_name] + + # Copy tar file to a temporary folder with retry logic + retries = 3 + temp_dir = tempfile.mkdtemp() + random_base_name = f"{uuid.uuid4().hex}.tar" + temp_tar_path = os.path.join(temp_dir, random_base_name) + for attempt in range(retries): + try: + shutil.copy(tar_path, temp_tar_path) + break + except Exception as e: + print(f"Failed to copy tar file {tar_path} after {retries} attempts: {e}") + if attempt == retries - 1: + shutil.rmtree(temp_dir) + raise RuntimeError(f"Failed to copy tar file {tar_path} after {retries} attempts: {e}. Abort") + time.sleep(1) + video_frames = {} + frame_idx_to_batch_idx = {} + repeat_happend = {} + try: + with tarfile.open(temp_tar_path, "r") as f: + for key in self.camkeys: + target_frame_idxs = [] + for view_idx, frame_idx in zip(view_idxs, frame_idxs): + if self.camkeys[view_idx] == key: + target_frame_idxs.append(frame_idx) + + if len(target_frame_idxs) == 0: + break + + diff = np.diff(np.array(target_frame_idxs)) + assert np.all(diff > 0), "frame_idxs are given as non-increasing list" + fp = f.extractfile(f"{clip_name}.{key}.mp4") + assert fp is not None, f"Video {clip_name}.{key}.mp4 not found" + vid = io.BytesIO(fp.read()) + vid.seek(0) + with av.open(vid) as input_container: + average_fps = input_container.streams.video[0].average_rate + time_base = input_container.streams.video[0].time_base + average_frame_duration = int(1 / average_fps / time_base) + frames = [] + frame_iterator = input_container.decode(video=0) + cur_frame_idx_to_batch_idx = {} + prev_target_pts = -1 + repeat_happend_for_key = 0 + for batch_idx, target_frame_number in enumerate(target_frame_idxs): + adjusted_target_frame_number = sync_to_original[key][target_frame_number] + target_pts = adjusted_target_frame_number * average_frame_duration + if prev_target_pts == target_pts: + frames.append(frames[-1]) + repeat_happend_for_key += 1 + cur_frame_idx_to_batch_idx[target_frame_number] = batch_idx + continue + for frame in frame_iterator: + # find the frame + if frame.pts == target_pts: + prev_target_pts = target_pts + break + cur_frame_idx_to_batch_idx[target_frame_number] = batch_idx + frames.append(torch.as_tensor(frame.to_rgb().to_ndarray())) + video_frames[key] = torch.stack(frames) + frame_idx_to_batch_idx[key] = cur_frame_idx_to_batch_idx + repeat_happend[key] = repeat_happend_for_key + del frame_iterator + vid.close() + fp.close() + del vid + + finally: + if os.path.exists(temp_tar_path): + os.remove(temp_tar_path) + shutil.rmtree(temp_dir) + + return video_frames, frame_idx_to_batch_idx, repeat_happend + + def _read_t5_and_meta(self, video_idx: int, view_idxs: List[int]) -> List[dict[str, Any]]: + clip_name = self.clip_names[video_idx] + data = {} + + for item in self.t5_index[clip_name]: + meta_data_file = os.path.join(self.t5_dirs, item.replace(".tar", ".json")) + with open(meta_data_file, "r") as f: + meta_data = json.load(f) + if clip_name in meta_data: + for view_idx in view_idxs: + if str(view_idx) in meta_data[clip_name]: + data[int(view_idx)] = {} + data[int(view_idx)]["meta_data"] = meta_data[clip_name][str(view_idx)] + + tar_file_path = os.path.join(self.t5_dirs, item) + retries = 3 + temp_dir = tempfile.mkdtemp() + + random_base_name = f"{uuid.uuid4().hex}.tar" + temp_tar_file_path = os.path.join(temp_dir, random_base_name) + + # Copy the tar file to a temporary location with retries + for attempt in range(retries): + try: + shutil.copy(tar_file_path, temp_tar_file_path) + break + except Exception as e: + if attempt == retries - 1: + shutil.rmtree(temp_dir) + raise RuntimeError(f"Failed to copy tar file after {retries} attempts: {e}") + time.sleep(1) + + try: + # Open the tar file from the temporary location + with tarfile.open(temp_tar_file_path, "r") as f: + file_buf = f.extractfile(f"{clip_name}.{self.camkeys[view_idx]}.bin") + if file_buf is None: + raise FileNotFoundError( + f"File {clip_name}.{self.camkeys[view_idx]}.bin not found in {item}" + ) + + with io.BytesIO(file_buf.read()) as fp: + data[int(view_idx)]["T5"] = pickle.load(fp) + + file_buf.close() + + finally: + # Ensure temporary tar file and directory are cleaned up + if os.path.exists(temp_tar_file_path): + os.remove(temp_tar_file_path) + shutil.rmtree(temp_dir) + + assert len(data) != 0, "No data was loaded from the T5 index." + return data + + def _get_basic_meta_data(self, video_idx: int): + if video_idx in self.meta_data_cache: + return self.meta_data_cache[video_idx] + + clip_name = self.clip_names[video_idx] + tar_path = self.tar_index[clip_name] + video_info = {} + + retries = 3 + temp_dir = tempfile.mkdtemp() + + random_base_name = f"{uuid.uuid4().hex}.tar" + temp_tar_path = os.path.join(temp_dir, random_base_name) + + for attempt in range(retries): + try: + shutil.copy(tar_path, temp_tar_path) + break + except Exception as e: + if attempt == retries - 1: + shutil.rmtree(temp_dir) + raise RuntimeError(f"Failed to copy tar file after {retries} attempts: {e}") + time.sleep(1) + + try: + # Open the tar file from the temporary location + with tarfile.open(temp_tar_path, "r") as f: + for key in self.camkeys: + fp = f.extractfile(f"{clip_name}.{key}.json") + if fp is None: + raise FileNotFoundError(f"Metadata file {clip_name}.{key}.json not found in {tar_path}") + + vidinfo = json.load(fp) + video_info[key] = vidinfo + fp.close() + + # Synchronize timestamps + min_t, max_t = -np.inf, np.inf + original_timestamps = {} + for key in self.camkeys: + timestamps = np.array([info["timestamp"] for info in video_info[key]]) + min_t = max(min_t, timestamps.min()) + max_t = min(max_t, timestamps.max()) + original_timestamps[key] = timestamps + ref_timestamps = original_timestamps[self.camkeys[0]] + ref_timestamps = ref_timestamps[ref_timestamps >= min_t] + ref_timestamps = ref_timestamps[ref_timestamps <= max_t] + sync_to_original = {} + for key in self.camkeys: + sync_to_original[key] = np.searchsorted(original_timestamps[key], ref_timestamps) + + finally: + # Ensure temporary tar file and directory are deleted + if os.path.exists(temp_tar_path): + os.remove(temp_tar_path) + shutil.rmtree(temp_dir) + + return video_info, sync_to_original + + def egomotion_alpamayo_parser_initial_frame(self, video_idx, timestamps): + clip_name = self.clip_names[video_idx] + tar_path = self.tar_index[clip_name] + + ed_config = egomotion_decoder.EgoMotionDecoderConfig( + decode_strategy="at_0_frame", prediction_start_offset_range=[0.0, 0.0] + ) + with tarfile.open(tar_path, "r") as f: + data = {} + data["egomotion.npz"] = f.extractfile(f"{clip_name}.egomotion.npz").read() + data["live_egomotion.npz"] = f.extractfile(f"{clip_name}.live_egomotion.npz").read() + calibration = self._extract_calibration_from_sample(f.extractfile(f"{clip_name}.rig.json")) + data["rig.json"] = f.extractfile(f"{clip_name}.rig.json").read() + + ego_data = egomotion_decoder.decode_egomotion(data, timestamps, ed_config) + + # flatten xyz and rotation into a 12-dim vector + xyz = ego_data["ego_future_xyz"] + rot = ego_data["ego_future_rot"] + # traj is of shape (B, N, 12) where N is the number of futuer ego poses + # default N = 64, 6.4 seconds, each one sampled at 0.1 sec interval. + traj = torch.cat([xyz, rot.flatten(2, 3)], dim=-1) + return traj, calibration["rig_raw"] + + def _read_raw_video( + self, + video_idx: int, + frame_idxs: List[int], + view_idxs: List[int], + data_fields: List[DataField], + ) -> dict[DataField, Any]: + video_info, sync_to_original = self._get_basic_meta_data(video_idx) + video_frames, frame_idx_to_batch_idx, repeat_happend = self._get_video_frames( + video_idx, view_idxs, frame_idxs, sync_to_original + ) + output_dict = {} + for data_field in data_fields: + if data_field == DataField.IMAGE_RGB: + rgb_list = [] + num_repeated_frames = [] + cur_view_idx = -1 + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + cam_key = self.camkeys[view_idx] + batch_idx = frame_idx_to_batch_idx[cam_key][frame_idx] + rgb_list.append(video_frames[cam_key][batch_idx].permute(2, 0, 1).float() / 255.0) + if cur_view_idx != view_idx: + num_repeated_frames.append(repeat_happend[cam_key]) + cur_view_idx = view_idx + output_dict[data_field] = torch.stack(rgb_list) + output_dict["num_repeated_frames"] = torch.FloatTensor(num_repeated_frames) + elif self.decode_traj and data_field == DataField.TRAJECTORY: + unique_view_idxs = list(set(view_idxs)) + view_count = view_idxs.count(unique_view_idxs[0]) + for view_idx in unique_view_idxs[1:]: + assert view_count == view_idxs.count(view_idx), ( + "trajectory datafield expects equal number of frames per view" + ) + + # choose one camera timestamps to compute trajectory + if self.trajectory_base_camera_index in view_idxs: + target_traj_cam_index = self.trajectory_base_camera_index + else: + target_traj_cam_index = random.sample(view_idxs, 1)[0] + timestamps = [] + for frame_idx, view_idx in zip(frame_idxs, view_idxs): + if view_idx == target_traj_cam_index: + # compute timestamps of the target base camera + cam_key = self.camkeys[view_idx] + t0_stamp = video_info[cam_key][sync_to_original[cam_key][frame_idx]]["timestamp"] + timestamps.append(t0_stamp) + if len(timestamps) == 0: + raise ValueError("view_idxs do not contain the base camera for trajectory computation.") + + # compute trajectory for the target base camera + traj, rig_info = self.egomotion_alpamayo_parser_initial_frame(video_idx, timestamps) + + traj = traj[0][..., :3] # use xyz + output_dict[data_field] = rearrange(traj, "t c -> (t c)") + output_dict["rig_info"] = rig_info + else: + raise NotImplementedError(f"Can't handle data field {data_field}") + return output_dict diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/__init__.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/__init__.py new file mode 100644 index 00000000..439c4c45 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataverse.utils.general import dataset_from_config, logger, print_rich_dict +from dataverse.utils.version import get_version + +# flake8: noqa: F401 diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/camera.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/camera.py new file mode 100644 index 00000000..06210064 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/camera.py @@ -0,0 +1,645 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import numpy as np +import torch + + +class Pose: + """ + A class of operations on camera poses (PyTorch tensors with shape [...,3,4]). + Each [3,4] camera pose takes the form of [R|t]. + """ + + def __call__(self, R=None, t=None): + # Construct a camera pose from the given R and/or t. + assert R is not None or t is not None + if R is None: + if not isinstance(t, torch.Tensor): + t = torch.tensor(t) + R = torch.eye(3, device=t.device).repeat(*t.shape[:-1], 1, 1) + elif t is None: + if not isinstance(R, torch.Tensor): + R = torch.tensor(R) + t = torch.zeros(R.shape[:-1], device=R.device) + else: + if not isinstance(R, torch.Tensor): + R = torch.tensor(R) + if not isinstance(t, torch.Tensor): + t = torch.tensor(t) + assert R.shape[:-1] == t.shape and R.shape[-2:] == (3, 3) + R = R.float() + t = t.float() + pose = torch.cat([R, t[..., None]], dim=-1) # [...,3,4] + assert pose.shape[-2:] == (3, 4) + return pose + + def invert(self, pose, use_inverse=False): + # Invert a camera pose. + R, t = pose[..., :3], pose[..., 3:] + R_inv = R.inverse() if use_inverse else R.transpose(-1, -2) + t_inv = (-R_inv @ t)[..., 0] + pose_inv = self(R=R_inv, t=t_inv) + return pose_inv + + def compose(self, pose_list): + # Compose a sequence of poses together. + # pose_new(x) = poseN o ... o pose2 o pose1(x) + pose_new = pose_list[0] + for pose in pose_list[1:]: + pose_new = self.compose_pair(pose_new, pose) + return pose_new + + def compose_pair(self, pose_a, pose_b): + # pose_new(x) = pose_b o pose_a(x) + R_a, t_a = pose_a[..., :3], pose_a[..., 3:] + R_b, t_b = pose_b[..., :3], pose_b[..., 3:] + R_new = R_b @ R_a + t_new = (R_b @ t_a + t_b)[..., 0] + pose_new = self(R=R_new, t=t_new) + return pose_new + + def scale_center(self, pose, scale): + """Scale the camera center from the origin. + 0 = R@c+t --> c = -R^T@t (camera center in world coordinates) + 0 = R@(sc)+t' --> t' = -R@(sc) = -R@(-R^T@st) = st + """ + R, t = pose[..., :3], pose[..., 3:] + pose_new = torch.cat([R, t * scale], dim=-1) + return pose_new + + def interpolate(self, pose_a, pose_b, alpha): + """Interpolate between two poses with Slerp. + Args: + pose_a (tensor [...,3,4]): Pose at time t=0. + pose_b (tensor [...,3,4]): Pose at time t=1. + alpha (tensor [...,1]): Interpolation parameter. + Returns: + pose (tensor [...,3,4]): Pose at time t. + """ + R_a, t_a = pose_a[..., :3], pose_a[..., 3:] + R_b, t_b = pose_b[..., :3], pose_b[..., 3:] + q_a = quaternion.R_to_q(R_a) # [...,4] + q_b = quaternion.R_to_q(R_b) # [...,4] + q_intp = quaternion.interpolate(q_a, q_b, alpha) # [...,4] + R_intp = quaternion.q_to_R(q_intp) # [...,3,3] + t_intp = (1 - alpha) * t_a + alpha * t_b # [...,3] + pose_intp = torch.cat([R_intp, t_intp], dim=-1) # [...,3,4] + return pose_intp + + def to_4x4(self, pose): + last_row = torch.tensor([0, 0, 0, 1], device=pose.device)[None, None].expand(pose.shape[0], 1, 4) + return torch.cat([pose, last_row], dim=-2) + + +class Lie: + """ + Lie algebra for SO(3) and SE(3) operations in PyTorch. + """ + + def so3_to_SO3(self, w): # [..., 3] + wx = self.skew_symmetric(w) + theta = w.norm(dim=-1)[..., None, None] + eye = torch.eye(3, device=w.device, dtype=torch.float32) + A = self.taylor_A(theta) + B = self.taylor_B(theta) + R = eye + A * wx + B * wx @ wx + return R + + def SO3_to_so3(self, R, eps=1e-7): # [..., 3, 3] + trace = R[..., 0, 0] + R[..., 1, 1] + R[..., 2, 2] + theta = ((trace - 1) / 2).clamp(-1 + eps, 1 - eps).acos_()[ + ..., None, None + ] % np.pi # ln(R) will explode if theta==pi + lnR = 1 / (2 * self.taylor_A(theta) + 1e-8) * (R - R.transpose(-2, -1)) # FIXME: wei-chiu finds it weird + w0, w1, w2 = lnR[..., 2, 1], lnR[..., 0, 2], lnR[..., 1, 0] + w = torch.stack([w0, w1, w2], dim=-1) + return w + + def se3_to_SE3(self, wu): # [...,3] + w, u = wu.split([3, 3], dim=-1) + wx = self.skew_symmetric(w) + theta = w.norm(dim=-1)[..., None, None] + eye = torch.eye(3, device=w.device, dtype=torch.float32) + A = self.taylor_A(theta) + B = self.taylor_B(theta) + C = self.taylor_C(theta) + R = eye + A * wx + B * wx @ wx + V = eye + B * wx + C * wx @ wx + Rt = torch.cat([R, (V @ u[..., None])], dim=-1) + return Rt + + def SE3_to_se3(self, Rt, eps=1e-8): # [...,3,4] + R, t = Rt.split([3, 1], dim=-1) + w = self.SO3_to_so3(R) + wx = self.skew_symmetric(w) + theta = w.norm(dim=-1)[..., None, None] + eye = torch.eye(3, device=w.device, dtype=torch.float32) + A = self.taylor_A(theta) + B = self.taylor_B(theta) + invV = eye - 0.5 * wx + (1 - A / (2 * B)) / (theta**2 + eps) * wx @ wx + u = (invV @ t)[..., 0] + wu = torch.cat([w, u], dim=-1) + return wu + + def skew_symmetric(self, w): + w0, w1, w2 = w.unbind(dim=-1) + zero = torch.zeros_like(w0) + wx = torch.stack( + [ + torch.stack([zero, -w2, w1], dim=-1), + torch.stack([w2, zero, -w0], dim=-1), + torch.stack([-w1, w0, zero], dim=-1), + ], + dim=-2, + ) + return wx + + def taylor_A(self, x, nth=10): + # Taylor expansion of sin(x)/x. + ans = torch.zeros_like(x) + denom = 1.0 + for i in range(nth + 1): + if i > 0: + denom *= (2 * i) * (2 * i + 1) + ans = ans + (-1) ** i * x ** (2 * i) / denom + return ans + + def taylor_B(self, x, nth=10): + # Taylor expansion of (1-cos(x))/x**2. + ans = torch.zeros_like(x) + denom = 1.0 + for i in range(nth + 1): + denom *= (2 * i + 1) * (2 * i + 2) + ans = ans + (-1) ** i * x ** (2 * i) / denom + return ans + + def taylor_C(self, x, nth=10): + # Taylor expansion of (x-sin(x))/x**3. + ans = torch.zeros_like(x) + denom = 1.0 + for i in range(nth + 1): + denom *= (2 * i + 2) * (2 * i + 3) + ans = ans + (-1) ** i * x ** (2 * i) / denom + return ans + + +class Quaternion: + def q_to_R(self, q): # [...,4] + # https://en.wikipedia.org/wiki/Rotation_matrix#Quaternion + qa, qb, qc, qd = q.unbind(dim=-1) + R = torch.stack( + [ + torch.stack([1 - 2 * (qc**2 + qd**2), 2 * (qb * qc - qa * qd), 2 * (qa * qc + qb * qd)], dim=-1), + torch.stack([2 * (qb * qc + qa * qd), 1 - 2 * (qb**2 + qd**2), 2 * (qc * qd - qa * qb)], dim=-1), + torch.stack([2 * (qb * qd - qa * qc), 2 * (qa * qb + qc * qd), 1 - 2 * (qb**2 + qc**2)], dim=-1), + ], + dim=-2, + ) + return R + + def R_to_q(self, R, eps=1e-6): # [...,3,3] + # https://en.wikipedia.org/wiki/Rotation_matrix#Quaternion + row0, row1, row2 = R.unbind(dim=-2) + R00, R01, R02 = row0.unbind(dim=-1) + R10, R11, R12 = row1.unbind(dim=-1) + R20, R21, R22 = row2.unbind(dim=-1) + t = R[..., 0, 0] + R[..., 1, 1] + R[..., 2, 2] + r = (1 + t + eps).sqrt() + qa = 0.5 * r + qb = (R21 - R12).sign() * 0.5 * (1 + R00 - R11 - R22 + eps).sqrt() + qc = (R02 - R20).sign() * 0.5 * (1 - R00 + R11 - R22 + eps).sqrt() + qd = (R10 - R01).sign() * 0.5 * (1 - R00 - R11 + R22 + eps).sqrt() + q = torch.stack([qa, qb, qc, qd], dim=-1) + return q + + def invert(self, q): # [...,4] + qa, qb, qc, qd = q.unbind(dim=-1) + norm = q.norm(dim=-1, keepdim=True) + q_inv = torch.stack([qa, -qb, -qc, -qd], dim=-1) / norm**2 + return q_inv + + def product(self, q1, q2): # [...,4] + q1a, q1b, q1c, q1d = q1.unbind(dim=-1) + q2a, q2b, q2c, q2d = q2.unbind(dim=-1) + hamil_prod = torch.stack( + [ + q1a * q2a - q1b * q2b - q1c * q2c - q1d * q2d, + q1a * q2b + q1b * q2a + q1c * q2d - q1d * q2c, + q1a * q2c - q1b * q2d + q1c * q2a + q1d * q2b, + q1a * q2d + q1b * q2c - q1c * q2b + q1d * q2a, + ], + dim=-1, + ) + return hamil_prod + + def apply(self, q, p): + out = self.product( + self.product(q, torch.cat((torch.zeros(p.shape[:-1] + (1,)), p), -1)), + self.invert(q), + ) + return out[..., 1:] + + def interpolate(self, q1, q2, alpha): # [...,4],[...,4],[...,1] + # https://en.wikipedia.org/wiki/Slerp + cos_angle = (q1 * q2).sum(dim=-1, keepdim=True) # [...,1] + flip = cos_angle < 0 + q1 = q1 * (~flip) - q1 * flip # [...,4] + theta = cos_angle.abs().acos() # [...,1] + slerp = (((1 - alpha) * theta).sin() * q1 + (alpha * theta).sin() * q2) / theta.sin() # [...,4] + return slerp + + +pose = Pose() +lie = Lie() +quaternion = Quaternion() + + +def to_hom(X): + # Get homogeneous coordinates of the input. + X_hom = torch.cat([X, torch.ones_like(X[..., :1])], dim=-1) + return X_hom + + +# Basic operations of transforming 3D points between world/camera/image coordinates. +def world2cam(X, pose): # [B,N,3] + X_hom = to_hom(X) + return X_hom @ pose.transpose(-1, -2) + + +def cam2img(X, cam_intr): + return X @ cam_intr.transpose(-1, -2) + + +def img2cam(X, cam_intr): + _dtype = cam_intr.dtype + X = X.float() + cam_intr = cam_intr.float() + result = X @ cam_intr.inverse().transpose(-1, -2) + return result.to(dtype=_dtype) + + +def cam2world(X, pose): + _dtype = pose.dtype + X = X.float() + pose = pose.float() + X_hom = to_hom(X) + pose_inv = Pose().invert(pose) + result = X_hom @ pose_inv.transpose(-1, -2) + return result.to(dtype=_dtype) + + +def angle_to_rotation_matrix(a, axis): + # Get the rotation matrix from Euler angle around specific axis. + roll = dict(X=1, Y=2, Z=0)[axis] + if isinstance(a, float): + a = torch.tensor(a) + zero = torch.zeros_like(a) + eye = torch.ones_like(a) + M = torch.stack( + [ + torch.stack([a.cos(), -a.sin(), zero], dim=-1), + torch.stack([a.sin(), a.cos(), zero], dim=-1), + torch.stack([zero, zero, eye], dim=-1), + ], + dim=-2, + ) + M = M.roll((roll, roll), dims=(-2, -1)) + return M + + +def get_center_and_ray(pose, intr, image_size): + """ + Args: + pose (tensor [3,4]/[B,3,4]): Camera pose. + intr (tensor [3,3]/[B,3,3]): Camera intrinsics. + image_size (list of int): Image size. + Returns: + center_3D (tensor [HW,3]/[B,HW,3]): Center of the camera. + ray (tensor [HW,3]/[B,HW,3]): Ray of the camera with depth=1 (note: not unit ray). + """ + assert pose.dtype == torch.float32 and intr.dtype == torch.float32, ( + f"pose and intr should be float32, got {pose.dtype} and {intr.dtype}" + ) + + H, W = image_size + # Given the intrinsic/extrinsic matrices, get the camera center and ray directions. + with torch.no_grad(): + # Compute image coordinate grid. + X, Y = get_pixel_grid(W, H, pose.device, normalized_coordinate=False) # [H,W] + xy_grid = torch.stack([X, Y], dim=-1).view(-1, 2) # [HW,2] + # Compute center and ray. + if len(pose.shape) == 3: + batch_size = len(pose) + xy_grid = xy_grid.repeat(batch_size, 1, 1) # [B,HW,2] + grid_3D = img2cam(to_hom(xy_grid), intr) # [HW,3]/[B,HW,3] + center_3D = torch.zeros_like(grid_3D) # [HW,3]/[B,HW,3] + # Transform from camera to world coordinates. + grid_3D = cam2world(grid_3D, pose) # [HW,3]/[B,HW,3] + center_3D = cam2world(center_3D, pose) # [HW,3]/[B,HW,3] + ray = grid_3D - center_3D # [B,HW,3] + return center_3D, ray + + +def get_pixel_grid(width: int, height: int, device: torch.device, normalized_coordinate: bool = False): + """Generate pixel grid given the image size. + + Args: + width (int): image width + height (int): image height + device (torch.device) + normalized_coordinate (bool, optional): normalized coordinate is between 0 and 1. Defaults to False. + + Returns: + torch.tensor: x,y pixel grid + """ + y_range = torch.arange(height, dtype=torch.float32, device=device).add_(0.5) + x_range = torch.arange(width, dtype=torch.float32, device=device).add_(0.5) + if normalized_coordinate: + y_range = y_range / height + x_range = x_range / width + y, x = torch.meshgrid(y_range, x_range, indexing="ij") # [H, W] + return x, y + + +def get_3D_points_from_dist( + center: torch.tensor, ray_unit: torch.tensor, dist: torch.tensor, multiple_samples_per_ray: bool = False +): + """Convert dist to 3D points in the world coordinate. + + Args: + center (torch.tensor): camer center in world coordinates, [..., 3] + ray_unit (torch.tensor): ray directions (unit vector), [..., 3] + dist (torch.tensor): distance along the ray, [..., 1] or [..., N_samples, 1] if sampling muliple points along rays + multiple_samples_per_ray (bool): If True, dist is [..., N_samples, 1] + + Returns: + torch.tensor: [..., 3] or [..., N_samples, 3] + """ + assert torch.allclose(ray_unit.norm(dim=-1), torch.ones_like(ray_unit.norm(dim=-1))), ( + f"ray_unit norm is not equal to 1, max {ray_unit.norm(dim=-1).max()} min {ray_unit.norm(dim=-1).min()}" + ) + if multiple_samples_per_ray: + assert len(dist.shape) == len(center.shape) + 1 + center, ray_unit = center[..., None, :], ray_unit[..., None, :] # [...,1,3] + else: + assert len(dist.shape) == len(center.shape), f"dist shape {dist.shape} center shape {center.shape}" + points_3D = center + ray_unit * dist # [...,3]/[...,N_samples,3] + return points_3D + + +def get_3D_points_from_depth( + center: torch.tensor, ray: torch.tensor, depth: torch.tensor, multiple_samples_per_ray: bool = False +): + """Convert depth to 3D points in the world coordinate. + NOTE: this function assuems the ray is NOT noramlized and returned directly from get_center_and_ray()!! + + Args: + center (torch.tensor): camer center in world coordinates, [..., 3] + ray (torch.tensor): ray directions (z component is 1), [..., 3] + depth (torch.tensor): z depth from camera center, [..., 1] or [..., N_samples, 1] if sampling muliple points along rays + multiple_samples_per_ray (bool): If True, depth is [..., N_samples, 1] + + Returns: + torch.tensor: [..., 3] or [..., N_samples, 3] + """ + if multiple_samples_per_ray: + assert len(depth.shape) == len(center.shape) + 1 + center, ray = center[..., None, :], ray[..., None, :] # [...,1,3] + else: + assert len(depth.shape) == len(center.shape) + points_3D = center + ray * depth # [...,3]/[...,N,3] + return points_3D + + +def convert_NDC(center, ray, intr, near=1): + # Shift camera center (ray origins) to near plane (z=1). + # (Unlike conventional NDC, we assume the cameras are facing towards the +z direction.) + center = center + (near - center[..., 2:]) / ray[..., 2:] * ray + # Projection. + cx, cy, cz = center.unbind(dim=-1) # [...,R] + rx, ry, rz = ray.unbind(dim=-1) # [...,R] + scale_x = intr[..., 0, 0] / intr[..., 0, 2] # [...] + scale_y = intr[..., 1, 1] / intr[..., 1, 2] # [...] + cnx = scale_x[..., None] * (cx / cz) + cny = scale_y[..., None] * (cy / cz) + cnz = 1 - 2 * near / cz + rnx = scale_x[..., None] * (rx / rz - cx / cz) + rny = scale_y[..., None] * (ry / rz - cy / cz) + rnz = 2 * near / cz + center_ndc = torch.stack([cnx, cny, cnz], dim=-1) # [...,R,3] + ray_ndc = torch.stack([rnx, rny, rnz], dim=-1) # [...,R,3] + return center_ndc, ray_ndc + + +def convert_NDC2(center, ray, intr): + # Similar to convert_NDC() but shift the ray origins to its own image plane instead of the global near plane. + # Also this version is much more interpretable. + scale_x = intr[..., 0, 0] / intr[..., 0, 2] # [...] + scale_y = intr[..., 1, 1] / intr[..., 1, 2] # [...] + # Get the metric image plane (i.e. new "center"): (sx*cx/cz, sy*cy/cz, 1-2/cz). + center = center + ray # This is the key difference. + cx, cy, cz = center.unbind(dim=-1) # [...,R] + image_plane = torch.stack([scale_x[..., None] * cx / cz, scale_x[..., None] * cy / cz, 1 - 2 / cz], dim=-1) + # Get the infinity plane: (sx*rx/rz, sy*ry/rz, 1). + rx, ry, rz = ray.unbind(dim=-1) # [...,R] + inf_plane = torch.stack([scale_x[..., None] * rx / rz, scale_y[..., None] * ry / rz, torch.ones_like(rz)], dim=-1) + # The NDC ray is the difference between the two planes, assuming t \in [0,1]. + ndc_ray = inf_plane - image_plane + return image_plane, ndc_ray + + +def rotation_distance(R1, R2, eps=1e-7): + # http://www.boris-belousov.net/2016/12/01/quat-dist/ + R_diff = R1 @ R2.transpose(-2, -1) + trace = R_diff[..., 0, 0] + R_diff[..., 1, 1] + R_diff[..., 2, 2] + angle = ((trace - 1) / 2).clamp(-1 + eps, 1 - eps).acos_() # numerical stability near -1/+1 + return angle + + +def get_oscil_novel_view_poses(N=60, angle=0.05, dist=5): + # Create circular viewpoints (small oscillations). + theta = torch.arange(N) / N * 2 * np.pi + R_x = angle_to_rotation_matrix((theta.sin() * angle).asin(), "X") + R_y = angle_to_rotation_matrix((theta.cos() * angle).asin(), "Y") + pose_rot = pose(R=R_y @ R_x) + pose_shift = pose(t=[0, 0, dist]) + pose_oscil = pose.compose([pose.invert(pose_shift), pose_rot, pose_shift]) + return pose_oscil + + +def cross_product_matrix(x): + """Matrix form of cross product opertaion. + + param x: [3,] tensor. + return: [3, 3] tensor representing the matrix form of cross product. + """ + return torch.tensor( + [ + [0, -x[2], x[1]], + [x[2], 0, -x[0]], + [ + -x[1], + x[0], + 0, + ], + ] + ) + + +def essential_matrix(poses): + """Compute Essential Matrix from a relative pose. + + param poses: [views, 3, 4] tensor representing relative poses. + return: [views, 3, 3] tensor representing Essential Matrix. + """ + r = poses[..., 0:3] + t = poses[..., 3] + tx = torch.stack([cross_product_matrix(tt) for tt in t], axis=0) + return tx @ r + + +def fundamental_matrix(poses, intr1, intr2): + """Compute Fundamental Matrix from a relative pose and intrinsics. + + param poses: [views, 3, 4] tensor representing relative poses. + intr1: [3, 3] tensor. Camera intrinsic of reference image. + intr2: [views, 3, 3] tensor. Camera Intrinsic of target image. + return: [views, 3, 3] tensor representing Fundamental Matrix. + """ + return intr2.inverse().transpose(-1, -2) @ essential_matrix(poses) @ intr1.inverse() + + +def get_ray_depth_plane_intersection(center, ray, depths): + """Compute the intersection of a ray with a depth plane. + Args: + center (tensor [B,HW,3]): Camera center of the target pose. + ray (tensor [B,HW,3]): Ray direction of the target pose. + depth (tensor [L]): The depth values from the source view (e.g. for MPI planes). + Returns: + intsc_points (tensor [B,HW,L,3]): Intersecting 3D points with the MPI. + """ + # Each 3D point x along the ray v from center c can be written as x = c+t*v. + # Plane equation: n@x = d, where normal n = (0,0,1), d = depth. + # --> t = (d-n@c)/(n@v). + # --> x = c+t*v = c+(d-n@c)/(n@v)*v. + center, ray = center[:, :, None], ray[:, :, None] # [B,HW,L,3], [B,HW,1,3] + depths = depths[None, None, :, None] # [1,1,L,1] + intsc_points = center + (depths - center[..., 2:]) / ray[..., 2:] * ray # [B,HW,L,3] + return intsc_points + + +def unit_view_vector_to_rotation_matrix(v, axes="ZYZ"): + """ + Args: + v (tensor [...,3]): Unit vectors on the view sphere. + axes: rotation axis order. + + Returns: + rotation_matrix (tensor [...,3,3]): rotation matrix R @ v + [0, 0, 1] = 0. + """ + alpha = torch.arctan2(v[..., 1], v[..., 0]) # [...] + beta = np.pi - v[..., 2].arccos() # [...] + euler_angles = torch.stack([torch.ones_like(alpha) * np.pi / 2, -beta, alpha], dim=-1) # [...,3] + rot2 = angle_to_rotation_matrix(euler_angles[..., 2], axes[2]) # [...,3,3] + rot1 = angle_to_rotation_matrix(euler_angles[..., 1], axes[1]) # [...,3,3] + rot0 = angle_to_rotation_matrix(euler_angles[..., 0], axes[0]) # [...,3,3] + rot = rot2 @ rot1 @ rot0 # [...,3,3] + return rot.transpose(-2, -1) + + +def sample_on_spherical_cap(anchor, N, max_angle, min_angle=0.0): + """Sample n points on the view hemisphere within the angle to x. + Args: + anchor (tensor [...,3]): Reference 3-D unit vector on the view hemisphere. + N (int): Number of sampled points. + max_angle (float): Sampled points should have max angle to x. + Returns: + sampled_points (tensor [...,N,3]): Sampled points on the spherical caps. + """ + batch_shape = anchor.shape[:-1] + # First, sample uniformly on a unit 2D disk. + radius = torch.rand(*batch_shape, N, device=anchor.device) # [...,N] + h_max = 1 - np.cos(max_angle) # spherical cap height + h_min = 1 - np.cos(min_angle) # spherical cap height + radius = (radius * (h_max - h_min) + h_min) / h_max + theta = torch.rand(*batch_shape, N, device=anchor.device) * 2 * np.pi # [...,N] + x = radius.sqrt() * theta.cos() # [...,N] + y = radius.sqrt() * theta.sin() # [...,N] + # Reparametrize to a unit spherical cap with height h. + # http://marc-b-reynolds.github.io/distribution/2016/11/28/Uniform.html + k = h_max * radius # [...,N] + s = (h_max * (2 - k)).sqrt() # [...,N] + points = torch.stack([s * x, s * y, 1 - k], dim=-1) # [...,N,3] + # Transform to center around the anchor. + ref_z = torch.tensor([0.0, 0.0, 1.0], device=anchor.device) + v = -anchor.cross(ref_z) # [...,3] + ss_v = lie.skew_symmetric(v) # [...,3,3] + R = torch.eye(3, device=anchor.device) + ss_v + ss_v @ ss_v / (1 + anchor @ ref_z)[..., None, None] # [...,3,3] + points = points @ R.transpose(-2, -1) # [...,N,3] + return points + + +def sample_on_spherical_cap_northern(anchor, N, max_angle, away_from=None, max_reject_count=None): + """Sample n points only the northern view hemisphere within the angle to x.""" + + def find_invalid_points(points): + southern = points[..., 2] < 0 # [...,N] + if away_from is not None: + cosine_ab = (away_from * anchor).sum(dim=-1, keepdim=True) # [...,1] + cosine_ac = (away_from[..., None, :] * points).sum(dim=-1) # [...,N] + not_outwards = cosine_ab < cosine_ac # [...,N] + invalid = southern | not_outwards + else: + invalid = southern + return invalid + + assert (anchor[..., 2] > 0).all() + assert anchor.norm(dim=-1).allclose(torch.ones_like(anchor[..., 0])) + points = sample_on_spherical_cap(anchor, N, max_angle) # [...,N,3] + invalid = find_invalid_points(points) + count = 0 + while invalid.any(): + # Reject and resample. + points_resample = sample_on_spherical_cap(anchor, N, max_angle) + points[invalid] = points_resample[invalid] + invalid = find_invalid_points(points) + count += 1 + if max_reject_count and count > max_reject_count: + points = anchor.repeat(N, 1) + return points + + +def dist_to_pointcloud(dist: torch.tensor, intr: torch.tensor, extr: torch.tensor): + """Convert dist along ray to pointcloud. + Args: + dist (torch.tensor): [1,H,W]/[B,1,H,W] + intr (torch.tensor): [3,3]/[B,3,3] + extr (torch.tensor): [3,4]/[B,3,4] + + Returns: + pc_dist (torch.tensor): [HW,3] + """ + + assert len(dist.shape) == len(intr.shape) + 1, ( + f"dist ({dist.shape}) and intr ({intr.shape}) should have the same batch size" + ) + # convert dist to pointcloud + center, ray = get_center_and_ray(extr, intr, dist.shape[-2:]) + ray = ray / ray.norm(dim=-1, keepdim=True) + dist = dist.view(*center.shape[:-1], 1) # [HW, 1]/[B,HW,1] + pc_dist = get_3D_points_from_dist(center, ray, dist) # HW,3/B,HW,3 + return pc_dist diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/constants.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/constants.py new file mode 100644 index 00000000..cfeb696b --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/constants.py @@ -0,0 +1,101 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Final + + +CROSS_LEFT_CAMERA_NAME: Final = "camera_cross_left_120fov" +CROSS_RIGHT_CAMERA_NAME: Final = "camera_cross_right_120fov" +FRONT_TELE_CAMERA_NAME: Final = "camera_front_tele_30fov" +FRONT_WIDE_CAMERA_NAME: Final = "camera_front_wide_120fov" +REAR_LEFT_CAMERA_NAME: Final = "camera_rear_left_70fov" +REAR_RIGHT_CAMERA_NAME: Final = "camera_rear_right_70fov" +REAR_TELE_CAMERA_NAME: Final = "camera_rear_tele_30fov" + +# Camera indices will be used for computing camera embeddings. +CAMERA_NAMES_TO_INDICES = { + CROSS_LEFT_CAMERA_NAME: 0, + FRONT_WIDE_CAMERA_NAME: 1, + CROSS_RIGHT_CAMERA_NAME: 2, + REAR_LEFT_CAMERA_NAME: 3, + REAR_TELE_CAMERA_NAME: 4, + REAR_RIGHT_CAMERA_NAME: 5, + FRONT_TELE_CAMERA_NAME: 6, +} +CAMERA_NAMES = tuple(name for name in CAMERA_NAMES_TO_INDICES) +CAMERA_INDICES_TO_NAMES = {idx: name for name, idx in CAMERA_NAMES_TO_INDICES.items()} + +# fps for v2 data. +VIDEO_FPS = 30 + +# minimum start index. +# NOTE V2 data might have issue in the first few frames, set a minimum to exclude them. +MIN_START_INDEX = 3 + +# Roughly-estimated car dimensions (length, width, height) in meters. +# For full accuracy, the actual ego-vehicle dimensions for each clip can be found +# in the rig.json file, as rig["rig"]["vehicle"]["value"]["body"]. +EGO_VEHICLE_LWH = (4.0, 3.0, 2.0) +# position of ego coordinate frame along the length of the car +# (distance of rear axle from back bumper / the length of the car) +# true value can be computed from rig.json file using +# -rig["rig"]["vehicle"]["value"]["body"]["boundingBoxPosition"][0] / ego_length +# we assume it is centered along the width, and so far are not using the z offset +EGO_VEHICLE_LENGTH_OFFSET = 0.21 + +# Mapping between raw object sub-categories loaded from the data and grouped object +# class categories used for training the detector +CLASS_TO_SUBCATEGORY_INV = { + "automobile": "Car", + "trailer": "Car", + "truck": "Truck", + "heavy_truck": "Truck", + "bus": "Truck", + "train_or_tram_car": "Truck", + "other_vehicle": "Truck", + "pedestrian": "Pedestrian", + "person": "Pedestrian", + "person_group": "Pedestrian", + "bicycle": "Cyclist", + "stroller": "Cyclist", + "bicycle_with_rider": "Cyclist", + "motorcycle_with_rider": "Cyclist", + "motorcycle": "Cyclist", + "rider": "Cyclist", + "cycle": "Others", + "protruding_object": "Others", + "animal": "Others", + "unknown": "Others", +} +CLASS_TO_SUBCATEGORY = { + "Car": "automobile", + "Truck": "truck", + "Pedestrian": "pedestrian", + "Cyclist": "bicycle", + "Others": "unknown", +} +CLASSES = list(CLASS_TO_SUBCATEGORY.keys()) +CLASSES_SUB = list(CLASS_TO_SUBCATEGORY_INV.keys()) + +LIDAR_NAME: Final = "lidar_gt_top_p128" +"""Identifier for the LiDAR sensor in the dataset.""" + +ALPACKAGES_BUCKET: Final = "alpackages" +"""The name of the S3/SwiftStack bucket/container where alpackages are stored.""" + +S3_REGION: Final = "us-east-1" +"""The region of the S3/SwiftStack bucket/container.""" + +S3_ENDPOINT_URL: Final = "https://pdx.s8k.io" +"""The endpoint URL for the S3/SwiftStack bucket/container.""" diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/egomotion_decoder.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/egomotion_decoder.py new file mode 100644 index 00000000..b0a06822 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/egomotion_decoder.py @@ -0,0 +1,759 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import io +import json +import random +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, TypedDict + +import numpy as np +import numpy.typing as npt +import pandas as pd +import scipy.spatial.transform as spt +import torch +from dataverse.utils import logger +from dataverse.utils.alpamayo import transformation +from dataverse.utils.alpamayo.camera import Quaternion +from scipy.interpolate import interp1d + + +_VALID_DECODE_STRATEGY_PATTERN = r"^(random|uniform|at)_(-?\d+)_frame$" + + +class EgoMotionData(TypedDict): + """Encompasses all required information to interpolate ego motion data.""" + + tmin: int + tmax: int + tparam: npt.NDArray[np.float32] + xyzs: npt.NDArray[np.float32] + quats: npt.NDArray[np.float32] + + +def parse_egopose_data(egopose_info: dict): + """Minimal parsing here. + + Args: + egopose_info: a dict containing the raw egopose data. + + Returns: + info (dict): a dict of numpy arrays with dtype object + """ + N, C = egopose_info["labels_data"].shape + (K,) = egopose_info["labels_keys"].shape + assert K == C, f"{K} {C}" + + info = {egopose_info["labels_keys"][ki]: egopose_info["labels_data"][:, ki] for ki in range(K)} + + # make sure sorted by time + assert np.all(0 < info["timestamp"][1:] - info["timestamp"][:-1]), info["timestamp"] + + return info + + +def adjust_orientation( + vals: npt.NDArray[np.float32] | torch.Tensor, +) -> npt.NDArray[np.float32] | torch.Tensor: + """Adjusts the orientation of the quaternions. + + Adjusts the orientation of the quaternions so that the dot product + between vals[i] and vals[i+1] is non-negative. + + Args: + vals (np.array or torch.tensor): (N, C) + + Returns: + vals (np.array or torch.tensor): (N, C) adjusted quaternions + """ + N, C = vals.shape + if isinstance(vals, torch.Tensor): + signs = torch.ones(N, dtype=vals.dtype, device=vals.device) + signs[1:] = torch.where(0 <= (vals[:-1] * vals[1:]).sum(dim=1), 1.0, -1.0) + signs = torch.cumprod(signs, dim=0) + + return vals * signs.reshape((N, 1)) + + else: + signs = np.ones(N, dtype=vals.dtype) + signs[1:] = np.where(0 <= (vals[:-1] * vals[1:]).sum(axis=1), 1.0, -1.0) + signs = np.cumprod(signs) + + return vals * signs.reshape((N, 1)) + + +def preprocess_egopose(poses: dict) -> EgoMotionData: + """Converts the poses to for interpolation. + + The dtype of all the inputs to the interpolator is float32. + TODO: instead of a linear interpolator for quaternions, + it'd be better to do slerp. + + Args: + poses (dict): a dict containing the raw egopose data. + + Returns: + A dictionary containing the following + tmin: int, the start time of the egopose data in microseconds + tmax: int, the end time of the egopose data in microseconds + tparam: list of floats, the relative (starting from 0) + timestamps of the egopose data in seconds + xyzs: list of lists of floats, the x,y,z position of the egopose + quats: list of lists of floats, the quaternion orientation of + the egopose + """ + # bounds of the interpolator as timestamps (ints) + tmin = poses["timestamp"][0] + tmax = poses["timestamp"][-1] + + # convert timestamps to float32 only after subtracting off tmin and + # converting from microseconds to seconds + tparam = (1e-6 * (poses["timestamp"] - tmin)).astype(np.float32) + + # prep x,y,z + # convert to float64, subtract off mean, convert to float32 + xyzs = np.stack( + ( + poses["x"].astype(np.float64), + poses["y"].astype(np.float64), + poses["z"].astype(np.float64), + ), + 1, + ) + xyzs = xyzs - xyzs.mean(axis=0, keepdims=True) + xyzs = xyzs.astype(np.float32) + + # prep quaternions + # parse directly as float32 + quats = np.stack( + ( + poses["qw"].astype(np.float32), + poses["qx"].astype(np.float32), + poses["qy"].astype(np.float32), + poses["qz"].astype(np.float32), + ), + 1, + ) + + # prep quaternions for interpolation https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L847 + # make sure normalized + quat_norm = np.linalg.norm(quats, axis=1) + EPS = 1e-3 + if not np.all(np.abs(quat_norm - 1.0) < EPS): + raise ValueError(f"Raw pose quaternions are too far from normalized; {quat_norm=}") + # adjust signs so that sequential dot product is always positive + quats = adjust_orientation(quats / quat_norm[:, None]) + + return EgoMotionData( + tmin=tmin, + tmax=tmax, + tparam=tparam, + xyzs=xyzs, + quats=quats, + ) + + +def load_dm_egopose(sample: dict) -> dict | None: + """Load egopose from raw tar files. + + Args: + sample: A dictionary containing the raw data to be decoded. + + Returns: + A dictionary containing the following keys: + tmin: int, the start time of the egopose data in microseconds + tmax: int, the end time of the egopose data in microseconds + tparam: list of floats, the relative (starting from 0) + timestamps of the egopose data in seconds + xyzs: list of lists of floats, the x,y,z position of the egopose + quats: list of lists of floats, the quaternion orientation of + the egopose + """ + ego_info = pd.read_parquet(io.BytesIO(sample["egomotion_estimate.parquet"]))["egomotion_estimate"] + ego_ts = pd.read_parquet(io.BytesIO(sample["clip.parquet"]))["key"][0]["time_range"] + ego_start_ts = ego_ts["start_micros"] + ego_end_ts = ego_ts["end_micros"] + + x = np.array([item["location"]["x"] for item in ego_info]) + y = np.array([item["location"]["y"] for item in ego_info]) + z = np.array([item["location"]["z"] for item in ego_info]) + + qx = np.array([item["orientation"]["x"] for item in ego_info]) + qy = np.array([item["orientation"]["y"] for item in ego_info]) + qz = np.array([item["orientation"]["z"] for item in ego_info]) + qw = np.array([item["orientation"]["w"] for item in ego_info]) + + timestamps = np.linspace(ego_start_ts, ego_end_ts, x.shape[0], endpoint=True) + timestamps = timestamps.astype(int) + + egopose_parsed = {} + egopose_parsed["coordinate_frame"] = "rig" + egopose_parsed["timestamp"] = timestamps + egopose_parsed["x"] = x + egopose_parsed["y"] = y + egopose_parsed["z"] = z + egopose_parsed["qw"] = qw + egopose_parsed["qx"] = qx + egopose_parsed["qy"] = qy + egopose_parsed["qz"] = qz + + ego_lerp_inp = preprocess_egopose(egopose_parsed) + return { + "tmin": ego_lerp_inp["tmin"], + "tmax": ego_lerp_inp["tmax"], + "tparam": ego_lerp_inp["tparam"].tolist(), + "xyzs": ego_lerp_inp["xyzs"].tolist(), + "quats": ego_lerp_inp["quats"].tolist(), + } + + +def load_egopose(sample: dict, live: bool = False, min_fps: int = 5) -> EgoMotionData | None: + """Load egopose from raw tar files. + + If we have fewer than 20 seconds, return None + + Args: + sample: A dictionary containing the raw data to be decoded. + live: if `True`, the "live" (estimated online) egomotion will be loaded + from the sample, otherwise the ground truth (optimized offline) + egomotion will be loaded. + min_fps: The minimum FPS of the egomotion data. + + Returns: + A dictionary containing the following keys: + tmin: int, the start time of the egopose data in microseconds + tmax: int, the end time of the egopose data in microseconds + tparam: list of floats, the relative (starting from 0) + timestamps of the egopose data in seconds + xyzs: list of lists of floats, the x,y,z position of the egopose + quats: list of lists of floats, the quaternion orientation of + the egopose + """ + pose_info = np.load( + io.BytesIO(sample["live_egomotion.npz" if live else "egomotion.npz"]), + allow_pickle=True, + ) + egopose_parsed = parse_egopose_data(pose_info) + + TMIN = 20.0 # seconds + egopose_span = 1e-6 * (egopose_parsed["timestamp"][-1] - egopose_parsed["timestamp"][0]) + if egopose_span < TMIN: + logger.warning(f"Insufficient egomotion data for this clip: {egopose_span=}") + return None + + # Check the FPS of egomotion data + delta = 1e-6 * (egopose_parsed["timestamp"][1:] - egopose_parsed["timestamp"][:-1]) + max_delta = 1.0 / min_fps + if not np.all(delta < max_delta): + logger.warning(f"Egomotion data does not meet frequency requirement: {max(delta)=}") + return None + + ego_lerp_inp = preprocess_egopose(egopose_parsed) + + coordinate_frame = egopose_parsed["coordinate_frame"][0].replace("_", ":") + if coordinate_frame != "rig": + # The logged egomotion is tracking a sensor's coordinate frame (e.g., the pose + # of the lidar) that is not the rig frame (origin at the rear axle center projected + # to ground, oriented front-left-up with respect to the vehicle's body). + # Here we use the "rig.json" to convert the sensor's pose to the rig's pose. + sensors = transformation.parse_rig_sensors_from_dict(json.loads(sample["rig.json"])) + if coordinate_frame not in sensors: + raise ValueError(f"Egomotion {coordinate_frame=} not found in rig.json.") + sensor_to_rig = transformation.sensor_to_rig(sensors[coordinate_frame]) + sensor_to_world_rots = spt.Rotation.from_quat(ego_lerp_inp["quats"], scalar_first=True) + # We derive rig_to_world by composing sensor_to_world and rig_to_sensor; + # with 4x4 rigid transformation matrices this would be: + # rig_to_world = sensor_to_world @ np.linalg.inv(sensor_to_rig) + rig_to_world_rots = sensor_to_world_rots * spt.Rotation.from_matrix(sensor_to_rig[:3, :3].T) + ego_lerp_inp["xyzs"] = ego_lerp_inp["xyzs"] - rig_to_world_rots.apply(sensor_to_rig[:3, 3]) + ego_lerp_inp["quats"] = adjust_orientation(rig_to_world_rots.as_quat(scalar_first=True)) + + return EgoMotionData( + tmin=ego_lerp_inp["tmin"], + tmax=ego_lerp_inp["tmax"], + tparam=ego_lerp_inp["tparam"].tolist(), + xyzs=ego_lerp_inp["xyzs"].tolist(), + quats=ego_lerp_inp["quats"].tolist(), + ) + + +class EgoPoseInterp: + """Interpolates egopose data.""" + + @classmethod + def load_from_raw_dict( + cls, + sample: Mapping[str, Any], + source: str = "online", + ): + """Init from a decoded data sample. + + Args: + sample: a decoded data sample. + source: the source of the egopose data, either "online" or "offline" or "deepmap". + """ + if source == "online": + ego_pose = load_egopose(sample, True) + elif source == "offline": + ego_pose = load_egopose(sample, False) + elif source == "deepmap": + ego_pose = load_dm_egopose(sample) + else: + raise ValueError(f"Invalid source: {source}") + if ego_pose is None: + raise ValueError("No ego pose available.") + return cls( + tmin=ego_pose["tmin"], + tmax=ego_pose["tmax"], + tparam=ego_pose["tparam"], + xyzs=ego_pose["xyzs"], + quats=ego_pose["quats"], + ) + + def __init__( + self, + tmin: int, + tmax: int, + tparam: npt.NDArray[np.float32], + xyzs: npt.NDArray[np.float32], + quats: npt.NDArray[np.float32], + ): + """Initialize the interpolator. + + Args: + tmin: int, the start time of the egopose data in microseconds + tmax: int, the end time of the egopose data in microseconds + tparam: list of floats, the relative (starting from 0) + timestamps of the egopose data in seconds + xyzs: list of lists of floats, the x,y,z position of the egopose + quats: list of lists of floats, the quaternion orientation of + the egopose + """ + self.tmin = tmin + self.tmax = tmax + + self.interp = interp1d( + tparam, + np.concatenate((xyzs, quats), 1), + kind="linear", + axis=0, + copy=False, + bounds_error=True, + assume_sorted=True, + ) + + def convert_tstamp(self, tstamp: int | npt.NDArray[np.int64]): + """Converts the absolute timestamp (microsecond) to relative (s).""" + return 1e-6 * (tstamp - self.tmin) + + def __call__( + self, t: npt.NDArray[np.float32], is_microsecond: bool = False + ) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.float32]]: + """Interpolate pose for t in seconds or microsecond.""" + EPS = 1e-5 + if is_microsecond: + t = self.convert_tstamp(t) + + out = self.interp(t) + xyzs = out[..., :3] + quats = out[..., 3:] + + # normalize quats + norm = np.linalg.norm(quats, axis=-1, keepdims=True) + assert np.all(EPS < norm), norm + quats = quats / norm + + return xyzs, quats + + +def _check_valid_tstamp_0( + tstamp_0: int, + prediction_start_offset_range: list[float], + ego_history_tvals: np.ndarray, + ego_future_tvals: np.ndarray, + history_ego_lerp_inp: dict, + future_ego_lerp_inp: dict, +) -> bool: + """Check if the tstamp_0 is valid for the given prediction_start_offset_range.""" + min_random_offset = int(prediction_start_offset_range[0] * 1e6) + max_random_offset = int(prediction_start_offset_range[1] * 1e6) + return ( + tstamp_0 + min_random_offset + int(ego_history_tvals[0] * 1e6) >= history_ego_lerp_inp["tmin"] + and tstamp_0 + max_random_offset + int(ego_history_tvals[-1] * 1e6) <= history_ego_lerp_inp["tmax"] + and tstamp_0 + min_random_offset + int(ego_future_tvals[0] * 1e6) >= future_ego_lerp_inp["tmin"] + and tstamp_0 + max_random_offset + int(ego_future_tvals[-1] * 1e6) <= future_ego_lerp_inp["tmax"] + ) + + +def interpolate_egopose( + ego_lerp_inp: dict | None, + live_ego_lerp_inp: dict | None, + prediction_start_offset_range: tuple[float, float], + ego_history_tvals: list[float], + ego_future_tvals: list[float], + base_timestamps: list[int], + decode_strategy: str, + num_route_points: int = 32, +) -> dict: + """Interpolates the egopose data starting at the certain timestamps. + + Taking the raw egopose data (ego_lerp_inp), we first decide starting time + indices (`ego_t0_frame_idx`) and then interpolate the egopose at the timestamps + base_timestamps[ego_t0_frame_idx] + trajectory_tvals. + + Args: + ego_lerp_inp (dict): A dictionary containing the complete egopose data (gt). + live_ego_lerp_inp (dict): A dictionary containing the complete live egopose data (live). + prediction_start_offset_range (tuple): range of possible relative + time offsets from last input image frame to prediction start time. + ego_history_tvals (list): the notional relative timestamps (i.e. t0 = 0.0s) of the + ego history data in seconds. + ego_future_tvals (list): the notional relative timestamps (i.e. t0 = 0.0s) of the + ego future data (i.e., ground truth for prediction) in seconds. + base_timestamps (list): the timestamps of the base frame + in microseconds (assume sorted). + decode_strategy: the strategy defines at which frames to decode the egopose. + valid strategies are: + - `random_N_frame`: randomly sample N frames from the available frames. + - `uniform_N_frame`: sample N uniformly spaced frames from the available frames. + - `at_N_frame`: sample the N-th frame from the available frames. + num_route_points: The number of points from egopose data to select as a temporary (TODO) + stand-in for route information. + + Returns: + A dictionary containing the following + ego_available (bool): whether the egopose data is available + ego_t0 (torch.tensor): (num_sample,) + the absolute start time of the egopose data in microseconds + ego_t0_relative (torch.tensor): (num_sample,) the relative start time of the egopose + data in seconds it is normalized to the first timestamp of the base_timestamps + ego_t0_frame_idx (torch.tensor): (num_sample,) the frame index of the base frame + prediction_start_offset (torch.tensor): (num_sample,) + the prediction start time offset + ego_history_tvals (torch.tensor): (Th,) + time in seconds corresponding to each position + ego_history_xyz (torch.tensor): (num_sample,Th,3) + the ego history (live) x,y,z positions + ego_history_rot (torch.tensor): (num_sample,Th,3,3) + the ego history (live) orientations as 3x3 matrices + ego_future_tvals (torch.tensor): (Tf,) + time in seconds corresponding to each position + ego_future_xyz (torch.tensor): (num_sample,Tf,3) + the ego future (gt) x,y,z positions + ego_future_rot (torch.tensor): (num_sample,Tf,3,3) + the ego future (gt) orientations as 3x3 matrices + route_xy (torch.tensor): (num_sample, num_route_points, 3) + the route x,y positions (from ego gt) + """ + # raise an error when we don't have ego pose data for the clip + if ego_lerp_inp is None: + raise ValueError("Invalid ego pose data for this clip.") + + if not all(x <= y for x, y in zip(base_timestamps[:-1], base_timestamps[1:])): + raise ValueError("base_timestamps is not sorted.") + + ego_lerp = EgoPoseInterp( + tmin=ego_lerp_inp["tmin"], + tmax=ego_lerp_inp["tmax"], + tparam=ego_lerp_inp["tparam"], + xyzs=ego_lerp_inp["xyzs"], + quats=ego_lerp_inp["quats"], + ) + + if live_ego_lerp_inp is None: + logger.warning("Using ego_lerp_inp in place of live_ego_lerp_inp (= None).") + live_ego_lerp_inp = ego_lerp_inp + live_ego_lerp = ego_lerp + else: + live_ego_lerp = EgoPoseInterp( + tmin=live_ego_lerp_inp["tmin"], + tmax=live_ego_lerp_inp["tmax"], + tparam=live_ego_lerp_inp["tparam"], + xyzs=live_ego_lerp_inp["xyzs"], + quats=live_ego_lerp_inp["quats"], + ) + + ego_history_tvals = np.array(ego_history_tvals, dtype=np.float32) + ego_future_tvals = np.array(ego_future_tvals, dtype=np.float32) + + match = re.match(_VALID_DECODE_STRATEGY_PATTERN, decode_strategy) + if match is None: + raise NotImplementedError(f"Decode strategy {decode_strategy} not implemented.") + strategy_type = match.group(1) + if strategy_type in {"random", "uniform"}: + # We work in timestamps (microseconds) to ensure temporal consistency between ego_lerp + # and live_ego_lerp (which have different notions of time t relative to their respective + # first timestamps). + valid_frame_indices = [ + ori + for ori, _tstamp_0 in enumerate(base_timestamps) + if _check_valid_tstamp_0( + tstamp_0=_tstamp_0, + prediction_start_offset_range=prediction_start_offset_range, + ego_history_tvals=ego_history_tvals, + ego_future_tvals=ego_future_tvals, + history_ego_lerp_inp=live_ego_lerp_inp, + future_ego_lerp_inp=ego_lerp_inp, + ) + ] + if len(valid_frame_indices) == 0: + raise ValueError( + "Insufficient ego pose data to fit history + future " + f"with maximum start offset {prediction_start_offset_range[1]}." + ) + num_frames = int(match.group(2)) + if num_frames > len(valid_frame_indices): + raise ValueError( + f"Requested {num_frames} frames, but only {len(valid_frame_indices)} frames are available." + ) + prediction_start_offset = np.random.uniform(*prediction_start_offset_range, size=num_frames) + if strategy_type == "random": + # sample randomly from timestamps for which history + future steps (shifted by + # prediction_start_offset) fit within the available ego data. + ego_t0_frame_idx = sorted(random.sample(valid_frame_indices, k=num_frames)) + else: + # sample uniformly from timestamps for which history + future steps (shifted by + # prediction_start_offset) fit within the available ego data. + # NOTE: we sample the last frame first to ensure that the last frame is included. + _step = len(valid_frame_indices) / num_frames + ego_t0_frame_idx = [valid_frame_indices[-(int(_step * i) + 1)] for i in range(num_frames)][::-1] + elif strategy_type == "at": + frame_idx = int(match.group(2)) + if frame_idx < 0: + ego_t0_frame_idx = [len(base_timestamps) + frame_idx] + else: + ego_t0_frame_idx = [frame_idx] + prediction_start_offset = np.random.uniform(*prediction_start_offset_range, size=1) + + tstamp_0 = [ + base_timestamps[_idx] + int(_start_offset * 1e6) + for _idx, _start_offset in zip(ego_t0_frame_idx, prediction_start_offset) + ] + # convert prediction-relative tvals to data sample tvals + # shape: (num_frames, num_history_steps) + history_live_ego_lerp_tvals = ( + np.array( + [live_ego_lerp.convert_tstamp(_tstamp_0) for _tstamp_0 in tstamp_0], + dtype=np.float32, + )[:, None] + + ego_history_tvals[None, :] + ) + # This check should only fail at "at" strategy + if not ( + live_ego_lerp_inp["tparam"][0] <= history_live_ego_lerp_tvals[:, 0].min() + and history_live_ego_lerp_tvals[:, -1].max() <= live_ego_lerp_inp["tparam"][-1] + ): + raise ValueError( + f"data: {live_ego_lerp_inp['tparam'][0]=} to {live_ego_lerp_inp['tparam'][-1]=}, " + f"while asking {history_live_ego_lerp_tvals[:, 0].min()=} to " + f"{history_live_ego_lerp_tvals[:, -1].max()=}" + ) + + # shape: (num_frames, num_future_steps) + future_future_ego_lerp_tvals = ( + np.array( + [ego_lerp.convert_tstamp(_tstamp_0) for _tstamp_0 in tstamp_0], + dtype=np.float32, + )[:, None] + + ego_future_tvals[None, :] + ) + # This check should only fail at "at" strategy + if not ( + ego_lerp_inp["tparam"][0] <= future_future_ego_lerp_tvals[:, 0].min() + and future_future_ego_lerp_tvals[:, -1].max() <= ego_lerp_inp["tparam"][-1] + ): + raise ValueError( + f"data: {ego_lerp_inp['tparam'][0]=} to {ego_lerp_inp['tparam'][-1]=}, " + f"while asking {future_future_ego_lerp_tvals[:, 0].min()=} to " + f"{future_future_ego_lerp_tvals[:, -1].max()=}" + ) + + # evaluate live pose at the history timesteps + ego_history_xyz, ego_history_quat = live_ego_lerp(history_live_ego_lerp_tvals) + # (num_frames, num_history_steps, 3) + ego_history_xyz = torch.tensor(ego_history_xyz, dtype=torch.float32) + # (num_frames, num_history_steps, 4) + ego_history_quat = torch.tensor(ego_history_quat, dtype=torch.float32) + + # transform coordinates to the ego's body frame (according to live) at the start of prediction + # (num_frames, 3), (num_frames, 3) + quaternion = Quaternion() + + live_xyz0, live_quat0 = live_ego_lerp( + np.array( + [live_ego_lerp.convert_tstamp(_tstamp_0) for _tstamp_0 in tstamp_0], + dtype=np.float32, + ) + ) + live_xyz0 = torch.tensor(live_xyz0, dtype=torch.float32) + live_inv_quat0 = quaternion.invert(torch.tensor(live_quat0, dtype=torch.float32)) + + ego_history_xyz = quaternion.apply(live_inv_quat0.unsqueeze(1), ego_history_xyz - live_xyz0.unsqueeze(1)) + ego_history_quat = quaternion.product(live_inv_quat0.unsqueeze(1), ego_history_quat) + + # TODO: remove duplicated code + # evaluate gt pose at the future timesteps + ego_future_xyz, ego_future_quat = ego_lerp(future_future_ego_lerp_tvals) + # (num_frames, num_future_steps, 3) + ego_future_xyz = torch.tensor(ego_future_xyz, dtype=torch.float32) + # (num_frames, num_future_steps, 3) + ego_future_quat = torch.tensor(ego_future_quat, dtype=torch.float32) + + # transform coordinates to the ego's body frame (according to gt) at the start of prediction + # (num_frames, 3), (num_frames, 3) + xyz0, quat0 = ego_lerp( + np.array( + [ego_lerp.convert_tstamp(_tstamp_0) for _tstamp_0 in tstamp_0], + dtype=np.float32, + ) + ) + xyz0 = torch.tensor(xyz0, dtype=torch.float32) + inv_quat0 = quaternion.invert(torch.tensor(quat0, dtype=torch.float32)) + ego_future_xyz = quaternion.apply(inv_quat0.unsqueeze(1), ego_future_xyz - xyz0.unsqueeze(1)) + ego_future_quat = quaternion.product(inv_quat0.unsqueeze(1), ego_future_quat) + + # infer ego route from gt pose and transform to ego's body frame at the start of prediction + # TODO: update route decoding when we have access to an alternative source + # (num_route_points, 3) + route_points = torch.tensor(ego_lerp_inp["xyzs"])[ + np.linspace(0, len(ego_lerp_inp["xyzs"]) - 1, num_route_points, dtype=int) + ] + route_xy = quaternion.apply(inv_quat0.unsqueeze(1), route_points.unsqueeze(0) - xyz0.unsqueeze(1))[..., :2] + + time_base = base_timestamps[0] + # make sure the relative_timestamp is slightly larger than original timestamp + # because later we will sort the timestamps of ego and images + # and we want to make sure the ego appears after the image + ego_t0_relative = [(t0 - time_base) * 1e-6 + 1e-5 for t0 in tstamp_0] + + return { + "ego_available": torch.tensor(True), + "ego_t0": torch.tensor(tstamp_0), + "ego_t0_relative": torch.tensor(ego_t0_relative), + "ego_t0_frame_idx": torch.tensor(ego_t0_frame_idx), + "prediction_start_offset": torch.from_numpy(prediction_start_offset).float(), + "ego_history_tvals": torch.from_numpy(ego_history_tvals).float(), + "ego_history_xyz": ego_history_xyz, + "ego_history_rot": quaternion.q_to_R(ego_history_quat), + "ego_future_tvals": torch.from_numpy(ego_future_tvals).float(), + "ego_future_xyz": ego_future_xyz, + "ego_future_rot": quaternion.q_to_R(ego_future_quat), + "route_xy": route_xy, + } + + +@dataclass(frozen=True) +class EgoMotionDecoderConfig: + """Configuration for the egomotion decoder. + + Attributes: + num_history_steps: number of history steps to load, i.e., with relative timestamps + `(-(num_history_steps - 1) * time_step, ..., -2 * time_step, -time_step, 0)` + to the prediction start time. + num_future_step: number of future steps to load, i.e., with relative timestamps + `(time_step, 2 * time_step, ..., num_future_steps * time_step)` + to the prediction start time. + time_step: time step (in seconds) between successive egomotion poses. + prediction_start_offset_range: min and max of possible relative time offsets from base image + frame to prediction start time. + force_base_frame_index: if not `None`, loaded trajectory is based from the specified + frame index in image_frames. (TODO: Deprecate this in favor of `decode_strategy`) + num_route_points: The number of points from egopose data to select as a temporary (TODO) + stand-in for route information. + decode_strategy: the strategy defines at which frames to decode the egopose. + valid strategies are: + - `random_N_frame`: randomly sample N frames from the available frames. + - `uniform_N_frame`: sample N uniformly spaced frames from the available frames. + - `at_N_frame`: sample the N-th frame from the available frames. + """ + + num_history_steps: int = 15 + num_future_steps: int = 64 + time_step: float = 0.1 + prediction_start_offset_range: tuple[float, float] = field(default_factory=lambda: (0.0, 1.5)) + force_base_frame_index: int | None = None + num_route_points: int = 32 + decode_strategy: str = "random_1_frame" + + def __post_init__(self): + """Make sure the config is valid.""" + if self.force_base_frame_index is not None: + logger.warning( + "force_base_frame_index is deprecating, using " + "`decode_strategy=at_%d_frame` instead." % self.force_base_frame_index + ) + self.decode_strategy = f"at_{self.force_base_frame_index}_frame" + if not re.match(_VALID_DECODE_STRATEGY_PATTERN, self.decode_strategy): + raise ValueError(f"Invalid decode strategy: {self.decode_strategy}") + + +def decode_egomotion( + data: dict, + base_timestamps: list[int], + config: EgoMotionDecoderConfig, +) -> dict: + """Decode egomotion from the data. + + Args: + data (dict): The raw data to be decoded. + it is assumed to contain the "egomotion.npz" (gt) and + "live_egomotion.npz" (live) fields + base_timestamps (list[int]): time stamps for each image frame + in microseconds. This is used to decide the 0-th timestamp of + the trajectory to load. + config: EgoMotionDecoderConfig + + Returns: + decoded_data (dict): containing the following fields: + ego_available (bool): whether the egopose data is available + ego_t0 (torch.tensor): (num_sample,) + the absolute start time of the egopose data in microseconds + ego_t0_relative (torch.tensor): (num_sample,) the relative start time of the egopose + data in seconds it is normalized to the first timestamp of the base_timestamps + ego_t0_frame_idx (torch.tensor): (num_sample,) the frame index of the base frame + prediction_start_offset (torch.tensor): (num_sample,) + the prediction start time offset + ego_history_tvals (torch.tensor): (Th,) + time in seconds corresponding to each position + ego_history_xyz (torch.tensor): (num_sample,Th,3) + the ego history (live) x,y,z positions + ego_history_rot (torch.tensor): (num_sample,Th,3,3) + the ego history (live) orientations as 3x3 matrices + ego_future_tvals (torch.tensor): (Tf,) + time in seconds corresponding to each position + ego_future_xyz (torch.tensor): (num_sample,Tf,3) + the ego future (gt) x,y,z positions + ego_future_rot (torch.tensor): (num_sample,Tf,3,3) + the ego future (gt) orientations as 3x3 matrices + route_xy (torch.tensor): (num_sample, config.num_route_points, 3) + the route x,y positions (from ego gt) + """ + ego_pose = load_egopose(data, live=False) + live_ego_pose = load_egopose(data, live=True) + ego_history_tvals = [config.time_step * t for t in range(-config.num_history_steps + 1, 1)] + ego_future_tvals = [config.time_step * t for t in range(1, config.num_future_steps + 1)] + return interpolate_egopose( + ego_lerp_inp=ego_pose, + live_ego_lerp_inp=live_ego_pose, + prediction_start_offset_range=config.prediction_start_offset_range, + ego_history_tvals=ego_history_tvals, + ego_future_tvals=ego_future_tvals, + base_timestamps=base_timestamps, + decode_strategy=config.decode_strategy, + num_route_points=config.num_route_points, + ) diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/generator_config.yaml b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/generator_config.yaml new file mode 100644 index 00000000..ea4d7b43 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/generator_config.yaml @@ -0,0 +1,101 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data: + camera_front_wide_120fov: + # 3840 × 2160 -> (bottom Crop + left right side crop) 3840 * 2160 -> (resize) 512 * 320 + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_cross_left_120fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_cross_right_120fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_front_tele_30fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_front_tele_sat_30fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_rear_tele_30fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_left_fisheye_200fov: + # H8 1936x1216, H81 1920x1536 + # Crop width to the smaller of the two dimensions, scale + crop_h: 1200 + crop_w: 1920 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_rear_fisheye_200fov: + # H8 1936x1216, H81 1920x1536 + crop_h: 1200 + crop_w: 1920 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_right_fisheye_200fov: + # H8 1936x1216, H81 1920x1536 + crop_h: 1200 + crop_w: 1920 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_front_fisheye_200fov: + # H8 1936x1216, H81 1920x1536 + crop_h: 1200 + crop_w: 1920 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_rear_right_70fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + camera_rear_left_70fov: + crop_h: 2160 + crop_w: 3840 + size_h: 1080 + size_w: 1920 + fps: 30 + + +conversion: + name: AV2fps-1080x512 + strict: T1920 + dataset_attr: labels + maxsize: 1e9 + maxcount: 4000 \ No newline at end of file diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/intrinsic_decoder.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/intrinsic_decoder.py new file mode 100644 index 00000000..09dcd4de --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/intrinsic_decoder.py @@ -0,0 +1,35 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataverse.utils.alpamayo.constants as constants +import dataverse.utils.alpamayo.ndas_camera_model as ndas_camera_model + + +def decode_intrinsic(rig_info: dict, camera_indices: list[int], raise_on_unsupported: bool = False): + """Decode intrinsics for those cameras in camera_indices.""" + outputs = {} + for name, sensor in rig_info.items(): + # if load with RANDOM, we will load all camera intrins to make the batching happy + name = name.replace(".mp4", "") + if name in constants.CAMERA_NAMES: + cam_id = constants.CAMERA_NAMES_TO_INDICES[name] + if cam_id in camera_indices: + if sensor["properties"]["polynomial-type"] != "pixeldistance-to-angle": + if raise_on_unsupported: + raise ValueError(f"{name} has an unsupported polynomial-type!") + else: + outputs[name] = None + else: + outputs[name] = ndas_camera_model.FThetaCamera.from_dict(sensor) + return outputs diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/ndas_camera_model.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/ndas_camera_model.py new file mode 100644 index 00000000..b6701eb5 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/ndas_camera_model.py @@ -0,0 +1,708 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import Any + +import dataverse.utils.alpamayo.constants as constants +import dataverse.utils.alpamayo.transformation as transformation +import numpy as np +from numpy.polynomial.polynomial import Polynomial +from scipy.optimize import curve_fit + + +class IdealPinholeCamera: + """Represents an ideal pinhole camera with no distortions. + + You can either pass in the fov or you can pass in the actual focal point parameters. It is the + users choice. If you pass in the fov, then the f_x, f_y parameters are computed for you. + Otherwise, they are directly inserted into the intrinsic matrix. + """ + + def __init__( + self, + fov_x_deg: float | int | None = None, + fov_y_deg: float | int | None = None, + f_x: float | int | None = None, + f_y: float | int | None = None, + width: int = 3848, + height: int = 2168, + ): + """The __init__ function. + + Args: + fov_x_deg (float | int | None): the horizontal FOV in degrees. + fov_y_deg (float | int | None): the vertical FOV in degrees. + f_x (float | int | None): the f_x value of the intrinsic calibration + matrix + f_y (float | int | None): the f_y value of the intrinsic calibration + matrix + width (int): the width of the image. Defaults to 3848 + height (int): the height of the image. Defaults to 2168 + """ + if f_x and fov_x_deg or f_y and fov_y_deg: + raise ValueError( + "Either f_x,f_y or fov_x_deg, fov_y_deg can" + "be passed in but not both. User must select which" + "operational mode you intend to use. If you want to" + "directly insert fx,fy into the intrinsic calibration" + "matrix then do not pass in fov_x_deg or fov_y_deg" + "and if you want to compute f_x, f_y from the FOV then" + "do not pass in f_x, f_y" + ) + + self._width = width + self._height = height + self._cx = width / 2 + self._cy = height / 2 + + # You can pass in the values directly. + if f_x and f_y: + self._f_x = f_x + self._f_y = f_y + else: + self._focal_from_fov(fov_x_deg, fov_y_deg) + + # The intrinsics matrix + self._k = np.asarray( + [[self._f_x, 0, self._cx], [0, self._f_y, self._cy], [0, 0, 1]], + dtype=np.float32, + ) + # The inverse of the intrinsics matrix (for backprojection) + self._k_inv = np.asarray( + [ + [1.0 / self._f_x, 0, -self._cx / self._f_x], + [0, 1.0 / self._f_y, -self._cy / self._f_y], + [0, 0, 1], + ], + dtype=np.float32, + ) + + @property + def width(self) -> int: + """Returns the width of the sensor.""" + return self._width + + @property + def height(self) -> int: + """Returns the height of the sensor.""" + return self._height + + def _focal_from_fov(self, fov_x_deg: float | int, fov_y_deg: float | int): + """Compute the focal length from horizontal and vertical FOVs. + + Args: + fov_x_deg (float | int): the horizontal FOV in degrees. + fov_y_deg (float | int): the vertical FOV in degrees. + """ + fov_x = np.radians(fov_x_deg) + fov_y = np.radians(fov_y_deg) + self._f_x = self._width / (2.0 * np.tan(fov_x * 0.5)) + self._f_y = self._height / (2.0 * np.tan(fov_y * 0.5)) + + def ray2pixel(self, rays: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Project 3D rays to 2D pixel coordinates. + + Args: + rays (np.ndarray): the rays as (N, 3) where N corresponds to + the number of rays and 3 is the (x,y,z) coordinates for each + ray. + + Returns: + projected (np.ndarray): Shape (N,2) the projected pixel coordinates + where N is the number of points and 2 corresponds to the (x,y) + dimensions. + valid (np.ndarray): of Shape (N,) the validity flag for each + projected pixel. Valid is a boolean array that can be used for + indexing rays that are within FOV. + """ + if np.ndim(rays) == 1: + rays = rays[np.newaxis, :] + + rays = rays.astype(np.float32) + + r = rays / rays[:, 2:] + + projected = np.matmul(self._k, r.T).T + + x_ok = np.logical_and(0 <= projected[:, 0], projected[:, 0] < self._width) + y_ok = np.logical_and(0 <= projected[:, 1], projected[:, 1] < self._height) + valid = np.logical_and(x_ok, y_ok) + return projected[:, :2], valid + + def pixel2ray(self, pixels: np.ndarray) -> np.ndarray: + """Backproject 2D pixels into 3D rays. + + Args: + pixels (np.ndarray): the pixels to backproject. + Size of (n_points, 2), where the first column contains + the `x` values, and the second column contains the `y` values. + + Returns: + rays (np.ndarray): the backprojected 3D rays. + """ + if np.ndim(pixels) == 1: + pixels = pixels[np.newaxis, :] + + pixels = pixels.astype(np.float32) + + # Add the third component of ones + pixels = np.c_[pixels, np.ones((pixels.shape[0], 1), dtype=np.float32)] + rays = np.matmul(self._k_inv, pixels.T).T + + # Normalize the rays + norm = np.linalg.norm(rays, axis=1, keepdims=True) + norm[norm == 0] = 1 + return rays / norm + + +class FThetaCamera: + """Defines an FTheta camera model.""" + + @classmethod + def from_rig(cls, rig_file: str, sensor_name: str): + """Initialize a new object using a rig file and the sensor's name. + + Args: + rig_file (str): the rig file path. + sensor_name (str): the name of the sensor. + + Returns: + FThetaCamera: the newly created object. + """ + with open(rig_file) as fp: + rig = json.load(fp) + + # Parse the properties from the rig file + sensors = rig["rig"]["sensors"] + sensor = None + sensor_found = False + + for sensor in sensors: + if sensor["name"] == sensor_name: + sensor_found = True + break + + if not sensor_found: + raise ValueError(f"The camera '{sensor_name}' was not found in the rig!") + + return cls.from_dict(sensor) + + @classmethod + def from_dict(cls, rig_dict: dict[str, Any]): + """Helper method to initialize a new object using a dictionary of the rig. + + Args: + rig_dict (dict): the sensor dictionary to initialize with. + + Returns: + FThetaCamera: the newly created object. + """ + ( + cx, + cy, + width, + height, + bw_poly, + ) = FThetaCamera.get_ftheta_parameters_from_json(rig_dict) + return cls(cx, cy, width, height, bw_poly) + + @classmethod + def from_intrinsics_array(cls, intrinsics: np.ndarray): + """Helper method to initialize a new object using an array of intrinsics. + + Args: + intrinsics (np.ndarray): the intrinsics array. The ordering is + expected to be "cx, cy, width, height, bw_poly". + This is the same ordering as the `intrinsics` + property of this class. + + Returns: + FThetaCaamera: the newly created object. + """ + return cls( + cx=intrinsics[0], + cy=intrinsics[1], + width=intrinsics[2], + height=intrinsics[3], + bw_poly=intrinsics[4:], + ) + + def __init__( + self, + cx: float, + cy: float, + width: int, + height: int, + bw_poly: np.ndarray, + focal_length_multiplier: float = 1.0, + target_width: int = 512, + target_height: int = 320, + ): + """The __init__ method. + + Args: + cx (float): optical center x. + cy (float): optical center y. + width (int): the width of the image. + height (int): the height of the image. + bw_poly (np.ndarray): the backward polynomial of the FTheta model. + focal_length_multiplier (float): the focal length multiplier, defaults to 1.0. + target_width (int): the target width of the rectified image, defaults to 512. + target_height (int): the target height of the rectified image, defaults to 320. + """ + self._center = np.asarray([cx, cy], dtype=np.float32) + self._width = int(width) + self._height = int(height) + self._target_width = int(target_width) + self._target_height = int(target_height) + self._focal_length_multiplier = focal_length_multiplier + self._bw_poly = Polynomial(bw_poly) + self._fw_poly = self._compute_fw_poly() + # Other properties that need to be computed + self._horizontal_fov = None + self._vertical_fov = None + self._max_angle = None + self._max_ray_angle = None + # Populate the array of intrinsics + self._intrinsics = np.append([cx, cy, width, height], bw_poly).astype(np.float32) + + self._update_calibrated_camera() + + ( + valid_source_pixels, + valid_target_pixels, + rectified_normalized_intrinsics, + target_resolution, + ) = self._compute_rectified_intrinsics( + self._width, + self._height, + self._target_width, + self._target_height, + self._focal_length_multiplier, + ) + + self.rectified_valid_source_pixels = valid_source_pixels + self.rectified_valid_target_pixels = valid_target_pixels + self.target_resolution = target_resolution + self._rectified_normalized_intrinsics = rectified_normalized_intrinsics + + @property + def rectified_normalized_intrinsics(self) -> np.ndarray: + """Obtain an array of the rectified intrinsics of this camera model. + + Returns: + np.ndarray: an array of rectified intrinsics. The ordering is + "cx, cy, width, height, bw_poly". dtype is np.float32. + """ + return self._rectified_normalized_intrinsics + + @staticmethod + def get_ftheta_parameters_from_json(rig_dict: dict[str, Any]) -> tuple[Any]: + """Helper method for obtaining FTheta camera model parameters. + + Args: + rig_dict (Dict[str, Any]): the rig dictionary to parse. + + Raises: + ValueError: if the provided rig is not supported. + AssertionError: if the provided model is supported, but cannot be + parsed properly. + + Returns: + Tuple[Any]: the values `cx`, `cy`, `width`, `height` and `bw_poly` + that were parsed. + """ + props = rig_dict["properties"] + + if props["Model"] != "ftheta": + raise ValueError("The given camera is not an FTheta camera") + + cx = float(props["cx"]) + cy = float(props["cy"]) + width = int(props["width"]) + height = int(props["height"]) + + if "bw-poly" in props: # Is this a regular rig? + poly = props["bw-poly"] + elif "polynomial" in props: # Is this a VT rig? + # VT rigs have a slightly different format, so need to handle these + # specifically. Refer to the following thread for more details: + # https://nvidia.slack.com/archives/C017LLEG763/p1633304770105300 + poly_type = props["polynomial-type"] + assert poly_type == "pixeldistance-to-angle", ( + "Encountered an unsupported VT rig. " + "Only `pixeldistance-to-angle` " + f"polynomials are supported (got {poly_type}). Rig:\n{rig_dict}" + ) + + linear_c = float(props["linear-c"]) if "linear-c" in props else None + linear_d = float(props["linear-d"]) if "linear-d" in props else None + linear_e = float(props["linear-e"]) if "linear-e" in props else None + + # If we had all the terms present, sanity check to make sure + # they are [1, 0, 0] + + if linear_c is not None and linear_d is not None and linear_e is not None: + if linear_c != 1.0: + raise ValueError(f"Expected `linear-c` term to be 1.0 (got {linear_c}. Rig:\n{rig_dict})") + if linear_d != 0.0: + raise ValueError(f"Expected `linear-d` term to be 1.0 (got {linear_d}. Rig:\n{rig_dict})") + if linear_e != 0.0: + raise ValueError(f"Expected `linear-e` term to be 1.0 (got {linear_e}. Rig:\n{rig_dict})") + + # If we're here, then it means we can parse the rig successfully. + poly = props["polynomial"] + else: + raise ValueError(f"Unable to parse the rig. Only FTheta rigs are supported! Rig:\n{rig_dict}") + + bw_poly = [np.float32(val) for val in poly.split()] + return cx, cy, width, height, bw_poly + + @property + def fov(self) -> tuple: + """Returns a tuple of horizontal and vertical fov of the sensor.""" + if self._vertical_fov is None or self._horizontal_fov is None: + self._compute_fov() + return self._horizontal_fov, self._vertical_fov + + @property + def width(self) -> int: + """Returns the width of the sensor.""" + return self._width + + @property + def height(self) -> int: + """Returns the height of the sensor.""" + return self._height + + @property + def center(self) -> np.ndarray: + """Returns the center of the sensor.""" + return self._center + + @property + def intrinsics(self) -> np.ndarray: + """Obtain an array of the intrinsics of this camera model. + + Returns: + np.ndarray: an array of intrinsics. The ordering is + "cx, cy, width, height, bw_poly". dtype is np.float32. + """ + return self._intrinsics + + def __str__(self): + """Returns a string representation of this object.""" + return ( + f"FTheta camera model:\n\t{self._bw_poly}\n\t" + f"center={self._center}\n\twidth={self._width}\n\theight={self._height}\n\t" + f"h_fov={np.degrees(self._horizontal_fov)}\n\tv_fov={np.degrees(self._vertical_fov)}" + ) + + def _update_calibrated_camera(self): + """Updates the internals of this object after calculating properties.""" + self._compute_fov() + self._max_ray_angle = (self._max_angle).copy() + is_fw_poly_slope_negative_in_domain = False + ray_angle = (np.float32(self._max_ray_angle)).copy() + deg2rad = np.pi / 180.0 + while ray_angle >= np.float32(0.0): + temp_dval = self._fw_poly.deriv()(self._max_ray_angle).item() + if temp_dval < 0: + is_fw_poly_slope_negative_in_domain = True + ray_angle -= deg2rad * np.float32(1.0) + + if is_fw_poly_slope_negative_in_domain: + ray_angle = (np.float32(self._max_ray_angle)).copy() + while ray_angle >= np.float32(0.0): + ray_angle -= deg2rad * np.float32(1.0) + raise ArithmeticError("FThetaCamera: derivative of distortion within image interior is negative") + + # Evaluate the forward polynomial at point (self._max_ray_angle, 0) + # Also evaluate its derivative at the same point + val = self._fw_poly(self._max_ray_angle).item() + dval = self._fw_poly.deriv()(self._max_ray_angle).item() + + if dval < 0: + raise ArithmeticError("FThetaCamera: derivative of distortion at edge of image is negative") + + self._max_ray_distortion = np.asarray([val, dval], dtype=np.float32) + + def _compute_rectified_intrinsics( + self, width, height, target_width, target_height, focal_length_multiplier=1.0 + ) -> np.ndarray: + normalized_centers = (self.center[0] / width, self.center[1] / height) + + target_resolution_w = round(target_width * focal_length_multiplier) + target_resolution_h = round(target_height * focal_length_multiplier) + target_resolution = (target_resolution_w, target_resolution_h) + target_principal_point = ( + target_resolution_w * normalized_centers[0], + target_resolution_h * normalized_centers[1], + ) + maglev_conf = transformation.get_video_parameters(constants.CAMERA_NAMES) + xbias, ybias, facw, _ = transformation.compute_preprocessing_transform(self, maglev_conf) + target_focal_length = ( + np.array( + [self._fw_poly.coef[1], self._fw_poly.coef[1]], + dtype=np.float32, + ) + / facw + ) + + # get the rectified pixel coords + # target resolution in width, height + target_pixels_x, target_pixels_y = np.meshgrid( + np.arange(target_resolution[0], dtype=np.int16), + np.arange(target_resolution[1], dtype=np.int16), + ) # [0, 1, 2, ..., w-1], [0, 1, 2, ..., h-1] + target_pixels = np.stack([target_pixels_x.flatten(), target_pixels_y.flatten()], axis=1) + target_image_points = target_pixels.astype(np.float32) + 0.5 + target_rays = (target_image_points - target_principal_point) / target_focal_length + target_rays = np.concatenate([target_rays, np.ones_like(target_rays[:, 0:1])], axis=1) + + source_pixels = self.ray2pixel(target_rays) + source_pixels[..., 0] -= xbias + source_pixels[..., 1] -= ybias + source_pixels /= facw + valid = ( + (source_pixels[..., 0] < target_width) + & (source_pixels[..., 0] >= 0.0) + & (source_pixels[..., 1] < target_height) + & (source_pixels[..., 1] >= 0.0) + ) + valid_source_pixels = source_pixels[valid].astype(np.int16) + valid_target_pixels = target_pixels[valid] + + rectified_normalized_intrinsics = np.array( + [ + target_focal_length[0] / target_resolution[0], + target_focal_length[1] / target_resolution[1], + target_principal_point[0] / target_resolution[0], + target_principal_point[1] / target_resolution[1], + ], + dtype=np.float32, + ) + + return ( + valid_source_pixels, + valid_target_pixels, + rectified_normalized_intrinsics, + target_resolution, + ) + + def _compute_fw_poly(self): + """Computes the forward polynomial for this camera. + + This function is a replication of the logic in the following file from + the DW repo: src/dw/calibration/cameramodel/CameraModels.cpp + """ + + def get_max_value(p0, p1): + return np.linalg.norm(np.asarray([p0, p1], dtype=self._center.dtype) - self._center) + + max_value = 0.0 + + size = (self._width, self._height) + value = get_max_value(0.0, 0.0) + max_value = max(max_value, value) + value = get_max_value(0.0, size[1]) + max_value = max(max_value, value) + value = get_max_value(size[0], 0.0) + max_value = max(max_value, value) + value = get_max_value(size[0], size[1]) + max_value = max(max_value, value) + + SAMPLE_COUNT = 500 + samples_x = [] + samples_b = [] + step = max_value / SAMPLE_COUNT + x = step + + for _ in range(0, SAMPLE_COUNT): + p = np.asarray([self._center[0] + x, self._center[1]], dtype=np.float32) + ray, _ = self.pixel2ray(p) + xy_norm = np.linalg.norm(ray[0, :2]) + theta = np.arctan2(float(xy_norm), float(ray[0, 2])) + samples_x.append(theta) + samples_b.append(float(x)) + x += step + + x = np.asarray(samples_x, dtype=np.float64) + y = np.asarray(samples_b, dtype=np.float64) + # Fit a 4th degree polynomial. The polynomial function is as follows: + + def f(x, b, x1, x2, x3, x4): + """4th degree polynomial.""" + return b + x * (x1 + x * (x2 + x * (x3 + x * x4))) + + # The constant in the polynomial should be zero, so add the `bounds` + # condition. + # FIXME(mmaghoumi) DW mentions disabling input normalization, + # what's that?? + # - the computation is more stable if the data is normalized before the + # fitting process. + coeffs, _ = curve_fit( + f, + x, + y, + bounds=( + [0, -np.inf, -np.inf, -np.inf, -np.inf], + [np.finfo(np.float64).eps, np.inf, np.inf, np.inf, np.inf], + ), + ) + # Return the polynomial and hardcode the bias value to 0 + return Polynomial([np.float32(val) if i > 0 else 0 for i, val in enumerate(coeffs)]) + + def pixel2ray(self, x: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Backproject 2D pixels into 3D rays. + + Args: + x (np.ndarray): the pixels to backproject. Size of (n_points, 2), + where the first column contains the `x` values, and the second + column contains the `y` values. + + Returns: + rays (np.ndarray): the backprojected 3D rays. Size of (n_points, 3). + valid (np.ndarray): bool flag indicating the validity of each + backprojected pixel. + """ + # Make sure x is n x 2 + if np.ndim(x) == 1: + x = x[np.newaxis, :] + + # Fix the type + x = x.astype(np.float32) + xd = x - self._center + xd_norm = np.linalg.norm(xd, axis=1, keepdims=True) + alpha = self._bw_poly(xd_norm) + sin_alpha = np.sin(alpha) + + rx = sin_alpha * xd[:, 0:1] / xd_norm + ry = sin_alpha * xd[:, 1:] / xd_norm + rz = np.cos(alpha) + + rays = np.hstack((rx, ry, rz)) + # special case: ray is perpendicular to image plane normal + valid = (xd_norm > np.finfo(np.float32).eps).squeeze() + rays[~valid, :] = (0, 0, 1) # This is what DW sets these rays to + + # note: + # if constant coefficient of bwPoly is non-zero, + # the resulting ray might not be normalized. + return rays, valid + + def ray2pixel(self, rays: np.ndarray) -> np.ndarray: + """Project 3D rays to 2D pixel coordinates. + + Args: + rays (np.ndarray): the rays. + + Returns: + result (np.ndarray): the projected pixel coordinates. + """ + # Make sure the input shape is (n_points, 3) + if np.ndim(rays) == 1: + rays = rays[np.newaxis, :] + + # Fix the type + rays = rays.astype(np.float32) + xy_norm = np.linalg.norm(rays[:, :2], axis=1, keepdims=True) + cos_alpha = rays[:, 2:] / np.linalg.norm(rays, axis=1, keepdims=True) + + alpha = np.empty_like(cos_alpha) + cos_alpha_condition = np.logical_and(cos_alpha > np.float32(-1.0), cos_alpha < np.float32(1.0)).squeeze() + alpha[cos_alpha_condition] = np.arccos(cos_alpha[cos_alpha_condition]) + alpha[~cos_alpha_condition] = xy_norm[~cos_alpha_condition] + + delta = np.empty_like(cos_alpha) + alpha_cond = alpha <= self._max_ray_angle + delta[alpha_cond] = self._fw_poly(alpha[alpha_cond]) + # For outside the model (which need to do linear extrapolation) + delta[~alpha_cond] = ( + self._max_ray_distortion[0] + (alpha[~alpha_cond] - self._max_ray_angle) * self._max_ray_distortion[1] + ) + + # Determine the bad points with a norm of zero, and avoid div by zero + bad_norm = xy_norm <= 0 + xy_norm[bad_norm] = 1 + delta[bad_norm] = 0 + # compute pixel relative to center + scale = delta / xy_norm + pixel = scale * rays + + # Handle the edge cases (ray along image plane normal) + edge_case_cond = (xy_norm <= np.float32(0.0)).squeeze() + pixel[edge_case_cond, :] = rays[edge_case_cond, :] + result = pixel[:, :2] + self._center + return result + + def _get_pixel_fov(self, pt: np.ndarray) -> float: + """Gets the FOV for a given point. Used internally for FOV computation. + + Args: + pt (np.ndarray): 2D pixel. + + Returns: + fov (float): the FOV of the pixel. + """ + ray, _ = self.pixel2ray(pt) + fov = np.arctan2(np.linalg.norm(ray[:, :2], axis=1), ray[:, 2]) + return fov + + def _compute_fov(self): + """Computes the FOV of this camera model.""" + max_x = self._width - 1 + max_y = self._height - 1 + + point_left = np.asarray([0, self._center[1]], dtype=np.float32) + point_right = np.asarray([max_x, self._center[1]], dtype=np.float32) + point_top = np.asarray([self._center[0], 0], dtype=np.float32) + point_bottom = np.asarray([self._center[0], max_y], dtype=np.float32) + + fov_left = self._get_pixel_fov(point_left) + fov_right = self._get_pixel_fov(point_right) + fov_top = self._get_pixel_fov(point_top) + fov_bottom = self._get_pixel_fov(point_bottom) + + self._vertical_fov = fov_top + fov_bottom + self._horizontal_fov = fov_left + fov_right + self._compute_max_angle() + + def _compute_max_angle(self): + """Computes the maximum ray angle for this camera.""" + max_x = self._width - 1 + max_y = self._height - 1 + + p = np.asarray([[0, 0], [max_x, 0], [0, max_y], [max_x, max_y]], dtype=np.float32) + + self._max_angle = max( # noqa: PLW3301 + max(self._get_pixel_fov(p[0, ...]), self._get_pixel_fov(p[1, ...])), + max(self._get_pixel_fov(p[2, ...]), self._get_pixel_fov(p[3, ...])), + ) + + def is_ray_inside_fov(self, ray: np.ndarray) -> bool: + """Determines whether a given ray is inside the FOV of this camera. + + Args: + ray (np.ndarray): the 3D ray. + + Returns: + bool: whether the ray is inside the FOV. + """ + if np.ndim(ray) == 1: + ray = ray[np.newaxis, :] + + ray_angle = np.arctan2(np.linalg.norm(ray[:, :2], axis=1), ray[:, 2]) + return ray_angle <= self._max_angle diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/rig_decoder.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/rig_decoder.py new file mode 100644 index 00000000..60a893f6 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/rig_decoder.py @@ -0,0 +1,25 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataverse.utils.alpamayo.transformation as transformation + + +def decode_rig_info(rig) -> dict: + """Decode rig information from json bytes.""" + # rig = json.loads(rig_json) + sensor_info = transformation.parse_rig_sensors_from_dict(rig) + vehicle_info = rig["rig"]["vehicle"] + result = {"_".join(name.split(":")): _ for name, _ in sensor_info.items()} + result["vehicle"] = vehicle_info + return result diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/rotation.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/rotation.py new file mode 100644 index 00000000..37e9c487 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/rotation.py @@ -0,0 +1,188 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +import numpy as np +import torch +from scipy.spatial.transform import Rotation as R + + +def so3_to_yaw_torch(rot_mat: torch.Tensor) -> torch.Tensor: + """Computes the yaw angle given an so3 rotation matrix (assumes that rotation is described in + xyz order) + + Args: + rot_mat (torch.Tensor): [..., 3,3] + + Returns: + torch.Tensor: [...] + """ + # phi is rotation about z, theta is rotation about y + cos_th_cos_phi = rot_mat[..., 0, 0] + cos_th_sin_phi = rot_mat[..., 1, 0] + return torch.atan2(cos_th_sin_phi, cos_th_cos_phi) + + +def so3_to_yaw_np(rot_mat: np.ndarray) -> np.ndarray: + """Computes the yaw angle given an so3 rotation matrix (assumes that rotation is described in + xyz order) + + Args: + rot_mat (np.ndarray): [..., 3,3] + + Returns: + np.ndarray: [...] + """ + cos_th_cos_phi = rot_mat[..., 0, 0] + cos_th_sin_phi = rot_mat[..., 1, 0] + return np.arctan2(cos_th_sin_phi, cos_th_cos_phi) + + +"""This file contains some utility functions for working with rotations.""" + + +def euler_2_so3(euler_angles: np.ndarray, degrees: bool = True, seq: str = "xyz") -> np.ndarray: + """Converts the euler angles representation to the so3 rotation matrix + Args: + euler_angles (np.array): euler angles [n,3] + degrees bool: True if angle is given in degrees else False + seq string: sequence in which the euler angles are given + + Out: + (np array): rotations given so3 matrix representation [n,3,3] + """ + return R.from_euler(seq=seq, angles=euler_angles, degrees=degrees).as_matrix().astype(np.float32) + + +def angle_wrap( + radians: Union[np.ndarray, torch.Tensor], +) -> Union[np.ndarray, torch.Tensor]: + """This function wraps angles to lie within [-pi, pi). + + Args: + radians (np.ndarray): The input array of angles (in radians). + + Returns: + np.ndarray: Wrapped angles that lie within [-pi, pi). + """ + return (radians + np.pi) % (2 * np.pi) - np.pi + + +def rotation_matrix(angle: Union[float, np.ndarray]) -> np.ndarray: + """Creates one or many 2D rotation matrices. + + Args: + angle (Union[float, np.ndarray]): The angle to rotate points by. + if float, returns 2x2 matrix + if np.ndarray, expects shape [...], and returns [...,2,2] array + + Returns: + np.ndarray: The 2x2 rotation matri(x/ces). + """ + batch_dims = 0 + if isinstance(angle, np.ndarray): + batch_dims = angle.ndim + + rotmat: np.ndarray = np.array( + [ + [np.cos(angle), -np.sin(angle)], + [np.sin(angle), np.cos(angle)], + ] + ) + return rotmat.transpose(*np.arange(2, batch_dims + 2), 0, 1) + + +def transform_coords_2d_np( + coords: np.ndarray, + offset: Optional[np.ndarray] = None, + angle: Optional[np.ndarray] = None, + rot_mat: Optional[np.ndarray] = None, +) -> np.ndarray: + """Args: + coords (np.ndarray): [..., 2] coordinates + offset (Optional[np.ndarray], optional): [..., 2] offset to translate. Defaults to None. + angle (Optional[np.ndarray], optional): [...] angle to rotate by. Defaults to None. + rot_mat (Optional[np.ndarray], optional): [..., 2,2] rotation matrix to apply. Defaults to None. + If rot_mat is given, angle is ignored. + + Returns: + np.ndarray: transformed coords + """ # noqa: E501 + if rot_mat is None and angle is not None: + rot_mat = rotation_matrix(angle) + + if rot_mat is not None: + coords = np.einsum("...ij,...j->...i", rot_mat, coords) + + if offset is not None: + coords += offset + + return coords + + +def stable_gramschmidt(M: torch.Tensor) -> torch.Tensor: + """Does gram-schmidt orthogonalization on 2 vectors in R3 across a batch. + + M is ...x3x2 + """ + EPS = 1e-7 + + x = M[..., 0] + y = M[..., 1] + x = x / torch.clamp_min(torch.norm(x, dim=-1, keepdim=True), EPS) + y = y - torch.sum(x * y, dim=-1, keepdim=True) * x + y = y / torch.clamp_min(torch.norm(y, dim=-1, keepdim=True), EPS) + z = torch.cross(x, y, dim=-1) + R = torch.stack((x, y, z), dim=-1) + return R + + +def rot_3d_to_2d(rot): + """Converts a 3D rotation matrix to a 2D rotation matrix by taking the x and y axes of the 3D + rotation matrix, projecting them to xy plan, and performing gram-schmidt orthogonalization. + + Args: + rot (torch.Tensor): The 3D rotation matrix to convert. + + Returns: + torch.Tensor: The 2D rotation matrix. + """ + xu = rot[..., :2, 0] + yu = rot[..., :2, 1] + EPS = 1e-6 + # gram-schmidt + xu = xu / (torch.norm(xu, dim=-1, keepdim=True) + EPS) + yu = yu - torch.sum(xu * yu, dim=-1, keepdim=True) * xu + yu = yu / (torch.norm(yu, dim=-1, keepdim=True) + EPS) + return torch.stack((xu, yu), dim=-1) + + +def rot_2d_to_3d(rot): + """Converts a 2D rotation matrix to a 3D rotation matrix assuming flat xy plane. + + Args: + rot (torch.Tensor): The 2D rotation matrix to convert. + + Returns: + torch.Tensor: The 3D rotation matrix. + """ + rot = torch.cat( + [ + torch.cat([rot, torch.zeros_like(rot[..., :1])], dim=-1), + torch.tensor([0.0, 0.0, 1.0], device=rot.device).repeat(rot.shape[:-2] + (1, 1)), + ], + dim=-2, + ) + return rot diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/transformation.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/transformation.py new file mode 100644 index 00000000..3199d518 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/transformation.py @@ -0,0 +1,324 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This file contains sensor transformation utilities, e.g., coordinate transforms from camera or +lidar frames to the ego-vehicle rig frame. + +""" + +import sys +from collections.abc import Iterable, Mapping +from typing import TYPE_CHECKING + +import numpy as np +import yaml +from pyquaternion import Quaternion + + +if TYPE_CHECKING: + from alpamayo.data import ndas_camera_model + +import dataverse.utils.alpamayo.rotation as rotation_utils + + +def get_sensor_to_sensor_flu(sensor: str) -> np.ndarray: + """Compute a rotation matrix that rotates sensor to Front-Left-Up format. + + Args: + sensor (str): sensor name. + + Returns: + np.ndarray: the resulting rotation matrix. + """ + if "cam" in sensor: + rot = [ + [0.0, 0.0, 1.0, 0.0], + [-1.0, 0.0, 0.0, 0.0], + [0.0, -1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ] + else: + rot = np.eye(4, dtype=np.float32) + + return np.asarray(rot, dtype=np.float32) + + +def parse_rig_sensors_from_dict(rig: dict) -> dict[str, dict]: + """Parses the provided rig dict into a dictionary indexed by sensor name. + + Args: + rig (Dict): Complete rig file as a dictionary. + + Returns: + (Dict): Dictionary of sensor rigs indexed by sensor name. + """ + # Parse the properties from the rig file + sensors = rig["rig"]["sensors"] + + sensors_dict = {sensor["name"]: sensor for sensor in sensors} + return sensors_dict + + +def sensor_to_rig(sensor: dict) -> np.ndarray | None: + """Obtain sensor to rig transformation matrix.""" + sensor_name = sensor["name"] + sensor_to_FLU = get_sensor_to_sensor_flu(sensor_name) + + if "nominalSensor2Rig_FLU" not in sensor: + # Some sensors (like CAN sensors) don't have an associated sensorToRig + return None + + nominal_T = sensor["nominalSensor2Rig_FLU"]["t"] + nominal_R = sensor["nominalSensor2Rig_FLU"]["roll-pitch-yaw"] + + correction_T = np.zeros(3, dtype=np.float32) + correction_R = np.zeros(3, dtype=np.float32) + + if "correction_rig_T" in sensor.keys(): + correction_T = sensor["correction_rig_T"] + + if "correction_sensor_R_FLU" in sensor.keys(): + assert "roll-pitch-yaw" in sensor["correction_sensor_R_FLU"].keys(), str(sensor["correction_sensor_R_FLU"]) + correction_R = sensor["correction_sensor_R_FLU"]["roll-pitch-yaw"] + + nominal_R = rotation_utils.euler_2_so3(nominal_R) + correction_R = rotation_utils.euler_2_so3(correction_R) + + R = nominal_R @ correction_R + T = np.array(nominal_T, dtype=np.float32) + np.array(correction_T, dtype=np.float32) + + transform = np.eye(4, dtype=np.float32) + transform[:3, :3] = R + transform[:3, 3] = T + + sensor_to_rig = transform @ sensor_to_FLU + + return sensor_to_rig + + +def traj2camera( + points: np.ndarray, + camera_xyz: np.ndarray, + camera_quat: np.ndarray, + camera_to_rig: np.ndarray, + camera_intrinsic: "ndas_camera_model.FThetaCamera", +): + """Project the trajectory points to the camera image. + + Args: + points: (N, 3) The trajectory points in global coordinate. + camera_xyz: (3,) The camera position in global coordinate. + camera_quat: (4,) The camera quaternion in global coordinate. + camera_to_rig: (4, 4) The camera extrinsics. + camera_intrinsic: The camera intrinsics. + + Returns: + uv: The projected points in pixel coordinate. + z: The depth of the points. + """ + ego_quat = Quaternion(camera_quat) + points = (points - camera_xyz) @ ego_quat.inverse.rotation_matrix.T + # closed-form inversion + rig_to_camera = np.zeros_like(camera_to_rig) + rig_to_camera[:3, :3] = camera_to_rig[:3, :3].T + rig_to_camera[:3, 3] = -camera_to_rig[:3, :3].T @ camera_to_rig[:3, 3] + camera_xyz = (points @ rig_to_camera[:3, :3].T) + rig_to_camera[:3, 3] + uv = camera_intrinsic.ray2pixel(camera_xyz) + z = camera_xyz[:, 2] + return uv, z + + +def box2camera( + points: np.ndarray, + camera_to_rig: np.ndarray, + camera_intrinsic: "ndas_camera_model.FThetaCamera", +) -> tuple[np.ndarray, np.ndarray]: + """Project the 3D bounding box to the camera image. + + Args: + points: (N, 3) The corner points of 3D bounding box in global coordinate. + camera_to_rig: (4, 4) The camera extrinsics. + camera_intrinsic: The camera intrinsics. + + Returns: + uv: The projected points in pixel coordinate. + z: The depth of the points. + """ + # closed-form inversion + rig_to_camera = np.zeros_like(camera_to_rig) + rig_to_camera[:3, :3] = camera_to_rig[:3, :3].T + rig_to_camera[:3, 3] = -camera_to_rig[:3, :3].T @ camera_to_rig[:3, 3] + camera_xyz = (points @ rig_to_camera[:3, :3].T) + rig_to_camera[:3, 3] + uv = camera_intrinsic.ray2pixel(camera_xyz) + z = camera_xyz[:, 2] + return uv, z + + +def get_traj_proj_on_image( + traj_xyz: np.ndarray, + camera_xyz: np.ndarray, + camera_quat: np.ndarray, + camera_intrinsic: "ndas_camera_model.FThetaCamera", + camera_extrinsics: np.ndarray, + maglev_conf: dict, + output_width: int | None = None, + output_height: int | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """Calculate the projection of trajectory points onto an image. + + Args: + traj_xyz (np.ndarray): (N, 3) Array of trajectory points in 3D space. + camera_xyz (np.ndarray): Camera position in 3D space. + camera_quat (np.ndarray): Camera orientation as a quaternion. + camera_intrinsic (ndas_camera_model.FThetaCamera): Camera intrinsic data. + camera_extrinsics (np.ndarray): Camera extrinsics matrix. + maglev_conf (dict): Configuration parameters for Maglev. + output_width (int): Width of the output image. TODO deprecate this for maglev_conf["size_w"] + output_height (int): Height of the output image. TODO deprecate this for + maglev_conf["size_h"] + + Returns: + np.ndarray: Array of UV coordinates representing the projection of trajectory points on + the image. + np.ndarray: (N,) Array of boolean values indicating the validity of the projection. + """ + uv, z = traj2camera( + points=traj_xyz, + camera_xyz=camera_xyz, + camera_quat=camera_quat, + camera_to_rig=camera_extrinsics, + camera_intrinsic=camera_intrinsic, + ) + # account for the cropping and resizing + xbias, ybias, facw, _ = compute_preprocessing_transform(camera_intrinsic, maglev_conf) + uv[:, 0] -= xbias + uv[:, 1] -= ybias + uv /= facw + + valid = ( + (z > 0) + & (uv[:, 0] < maglev_conf["size_w"] - 0.5) + & (uv[:, 0] > -0.5) + & (uv[:, 1] < maglev_conf["size_h"] - 0.5) + & (uv[:, 1] > -0.5) + ) + uv = uv[valid] + return uv, valid + + +def project_points_to_image( + points: np.ndarray, + camera_to_rig: np.ndarray, + camera_intrinsic: "ndas_camera_model.FThetaCamera", + maglev_processing_config: Mapping[str, float], + output_width: int = sys.maxsize, + output_height: int = sys.maxsize, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Project 3D points to an image. + + Args: + points: 3D points to project. Shape (N, 3). + camera_to_rig: Coordinate transformation from camera to rig frame. Shape (4, 4). + camera_intrinsic: Camera intrinsic matrix. + maglev_processing_config: Information about the preprocessing applied to the image, e.g., + cropping and resizing. + output_width: Width of the output image. + output_height: Height of the output image. + + Returns: + uv: The UV (i.e. image) coordinates of the projected points. + z: The depth of the projected points in camera space. + mask: A boolean mask indicating if the projection is valid. + """ + uv, z = box2camera(points, camera_to_rig, camera_intrinsic) + + # account for the cropping and resizing and apply the processing corrections + xbias, ybias, facw, _ = compute_preprocessing_transform(camera_intrinsic, maglev_processing_config) + uv[:, 0] -= xbias + uv[:, 1] -= ybias + uv /= facw + + xmask = np.logical_and(-0.5 < uv[:, 0], uv[:, 0] < output_width - 0.5) + ymask = np.logical_and(-0.5 < uv[:, 1], uv[:, 1] < output_height - 0.5) + mask = np.logical_and.reduce([xmask, ymask, z > 0]) + return uv, z, mask + + +def compute_preprocessing_transform( + camera_intrinsic: "ndas_camera_model.FThetaCamera", maglev_conf: dict +) -> tuple[float, float]: + """Pre-processing due to crop and resize. + + Args: + camera_intrinsic: The FTheta camera intrinsic. + maglev_conf: The maglev configuration from generator_config.yaml. + + Returns: + xbias: The x-bias due to the crop. + ybias: The y-bias due to the crop. + facw: The factor to rescale width. + fach: The factor to rescale height. + """ + og_width = int(camera_intrinsic.width) + og_height = int(camera_intrinsic.height) + + # TODO in the calibration comments, it seems like the + # original size is 3840x2160? But actually sometimes the + # calibration says the original size is 3848x2168 + + # the pre-processing steps are + # 1) bottom crop + # 2) symmetric left-right crop + # 3) rescaling + + # for transformation parameters, the bottom crop + # only changes the extent. So in terms of the + # change to the coordinate frame, we only take + # into account the left-right crop and rescaling. + + xbias = (og_width - maglev_conf["crop_w"]) / 2 + ybias = (og_height - maglev_conf["crop_h"]) / 2 + # make sure resize is equal + facw = maglev_conf["crop_w"] / maglev_conf["size_w"] + fach = maglev_conf["crop_h"] / maglev_conf["size_h"] + assert facw == fach, f"{facw} {fach}" + return xbias, ybias, facw, fach + + +def get_video_parameters( + camkeys: Iterable[str], + datapath: str | None = None, +) -> dict[str, int]: + """Load the pre-processing config file for the dataset. + + Currently we assert that all cameras use the same parameters with the only difference of + size_w/h, we could loosen that in the future. + """ + collect = {"crop_h": [], "crop_w": [], "size_h": [], "size_w": [], "fps": []} + if datapath is None: + datapath = __file__.replace("transformation.py", "generator_config.yaml") + with open(datapath) as reader: + conf = yaml.safe_load(reader) + + for camkey in camkeys: + for key in collect: + collect[key].append(conf["data"][camkey][key]) + # make sure they're all the same + for key in collect: + assert len(set(collect[key])) == 1, collect[key] + + # select the first one since they're all the same + out = {k: v[0] for k, v in collect.items()} + + return out diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/vis_trajectory.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/vis_trajectory.py new file mode 100644 index 00000000..798136e7 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/alpamayo/vis_trajectory.py @@ -0,0 +1,129 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Final + +import dataverse.utils.alpamayo.ndas_camera_model as ndas_camera_model +import dataverse.utils.alpamayo.rig_decoder as rig_decoder +import dataverse.utils.alpamayo.transformation as transformation +import matplotlib.pyplot as plt +import numpy as np +import torch +from matplotlib.axes import Axes + + +# Used for different predicted trajectories +# It comes from [rgb_to_hex(k) for k in plotly.colors.qualitative.Set3] +TRAJ_MODE_COLOR_MAP: Final[list[str]] = [ + "#8dd3c7", + "#ffffb3", + "#bebada", + "#fb8072", + "#80b1d3", + "#fdb462", + "#b3de69", + "#fccde5", + "#d9d9d9", + "#bc80bd", + "#ccebc5", + "#ffed6f", +] + + +def plot_traj_on_image( + ax: Axes, + xyzs: list[np.ndarray], + camera_intrinsic: ndas_camera_model.FThetaCamera, + camera_extrinsics: np.ndarray, + maglev_conf: dict[str, int], + camera_xyz: np.ndarray = np.zeros(3, dtype=np.float32), + camera_quat: np.ndarray = np.array([1, 0, 0, 0]), + colors: list[str] | None = None, + size: float = 1.0, +): + """Project trajectories onto 2D image and plot with different colors.""" + num_traj = len(xyzs) + assert xyzs[0].ndim == 2 + if colors is None: + colors = [TRAJ_MODE_COLOR_MAP[i % 12] for i in range(num_traj)] + else: + assert len(colors) == num_traj + + for i in range(num_traj): + traj_xyz = xyzs[i] + uv, _ = transformation.get_traj_proj_on_image( + traj_xyz=traj_xyz, + camera_xyz=camera_xyz, + camera_quat=camera_quat, + camera_extrinsics=camera_extrinsics, + camera_intrinsic=camera_intrinsic, + maglev_conf=maglev_conf, + ) + ax.scatter(uv[:, 0], uv[:, 1], s=size, c=colors[i]) + + # # NOTE we can use following code to draw a patch using car width. + # # Plot the shadow + # ax.fill_between(uv[:, 0], uv[:, 1] - 0.2, uv[:, 1] + 0.2, color='blue', alpha=0.3) + # # Plot the main blue line + # ax.plot(uv[:, 0], uv[:, 1], color='blue', linewidth=4) + ax.axis("off") + + +def render_image_with_traj( + ax: Axes, + traj_xyzs: torch.Tensor | None, + camera_image: torch.Tensor, + rig_info: dict, + cam_name: str = "camera_front_wide_120fov", +) -> plt.figure: + """Plot trajectory on 2D image. + + Args: + traj_xyzs (torch.Tensor): [K, N, 3], the origin should be on rig at camera_image's + timestamp. + camera_image (torch.Tensor): [3, H, W], float in 0-1 range. + rig_info (str | bytes): file path for rig info or the raw bytes data. + cam_name (str, optional): camera name used to retrieve camera parameters wrt rig. Defaults + to "camera_front_wide_120fov". + + Returns: + plt.figure: figure handler. + """ + C, H, W = camera_image.shape + assert C == 3 + assert traj_xyzs.ndim == 3 + + rig = rig_info + info = rig_decoder.decode_rig_info(rig) + + camera_image_np = camera_image.permute(1, 2, 0).numpy() + + camera_extrinsics: torch.Tensor = torch.from_numpy(transformation.sensor_to_rig(info[cam_name])) + camera_intrinsic = ndas_camera_model.FThetaCamera.from_dict(info[cam_name]) + + maglev_conf = transformation.get_video_parameters([cam_name]) + # updating size_h and size_w in maglev_conf in case the inputs image is resize to different + # ones than what's in maglev. + maglev_conf["size_h"] = H + maglev_conf["size_w"] = W + + ax.imshow(camera_image_np) + + plot_traj_on_image( + ax, + traj_xyzs.numpy(), + camera_intrinsic=camera_intrinsic, + camera_extrinsics=camera_extrinsics, + maglev_conf=maglev_conf, + ) diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/colmap.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/colmap.py new file mode 100644 index 00000000..b45c308c --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/colmap.py @@ -0,0 +1,139 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import collections +import struct + +import numpy as np + + +CameraModel = collections.namedtuple("CameraModel", ["model_id", "model_name", "num_params"]) +Camera = collections.namedtuple("Camera", ["id", "model", "width", "height", "params"]) +BaseImage = collections.namedtuple("Image", ["id", "qvec", "tvec", "camera_id", "name", "xys", "point3D_ids"]) + + +class Image(BaseImage): + def qvec2rotmat(self): + return qvec2rotmat(self.qvec) + + +CAMERA_MODELS = { + CameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3), + CameraModel(model_id=1, model_name="PINHOLE", num_params=4), + CameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4), + CameraModel(model_id=3, model_name="RADIAL", num_params=5), + CameraModel(model_id=4, model_name="OPENCV", num_params=8), + CameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8), + CameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12), + CameraModel(model_id=7, model_name="FOV", num_params=5), + CameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4), + CameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5), + CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12), +} +CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) for camera_model in CAMERA_MODELS]) + + +def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"): + """Read and unpack the next bytes from a binary file. + :param fid: + :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + :param endian_character: Any of {@, =, <, >, !} + :return: Tuple of read and unpacked values. + """ + data = fid.read(num_bytes) + return struct.unpack(endian_character + format_char_sequence, data) + + +def load_colmap_data(camdata, imdata): + list_of_keys = list(camdata.keys()) + cam = camdata[list_of_keys[0]] + + h, w, f = cam.height, cam.width, cam.params[0] + hwf = np.array([h, w, f]).reshape([3, 1]) + + w2c_mats = [] + bottom = np.array([0, 0, 0, 1.0]).reshape([1, 4]) + + names = [imdata[k].name for k in imdata] + perm = np.argsort(names) + for k in imdata: + im = imdata[k] + R = im.qvec2rotmat() + t = im.tvec.reshape([3, 1]) + m = np.concatenate([np.concatenate([R, t], 1), bottom], 0) + w2c_mats.append(m) + + w2c_mats = np.stack(w2c_mats, 0) + c2w_mats = np.linalg.inv(w2c_mats) + + poses = c2w_mats[:, :3, :4].transpose([1, 2, 0]) + poses = np.concatenate([poses, np.tile(hwf[..., np.newaxis], [1, 1, poses.shape[-1]])], 1) + + # must switch to [-u, r, -t] from [r, -u, t], NOT [r, u, -t] + poses = np.concatenate( + [ + poses[:, 1:2, :], + poses[:, 0:1, :], + -poses[:, 2:3, :], + poses[:, 3:4, :], + poses[:, 4:5, :], + ], + 1, + ) + return poses, perm, c2w_mats, w2c_mats + + +def qvec2rotmat(qvec): + return np.array( + [ + [ + 1 - 2 * qvec[2] ** 2 - 2 * qvec[3] ** 2, + 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], + 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2], + ], + [ + 2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], + 1 - 2 * qvec[1] ** 2 - 2 * qvec[3] ** 2, + 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1], + ], + [ + 2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], + 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], + 1 - 2 * qvec[1] ** 2 - 2 * qvec[2] ** 2, + ], + ] + ) + + +def rotmat2qvec(R): + Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat + K = ( + np.array( + [ + [Rxx - Ryy - Rzz, 0, 0, 0], + [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0], + [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0], + [Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz], + ] + ) + / 3.0 + ) + eigvals, eigvecs = np.linalg.eigh(K) + qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)] + if qvec[0] < 0: + qvec *= -1 + return qvec diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/general.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/general.py new file mode 100644 index 00000000..2a2dcd8d --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/general.py @@ -0,0 +1,115 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import logging + +import numpy as np +from omegaconf.dictconfig import DictConfig +from rich.console import Console +from rich.logging import RichHandler +from rich.table import Table + +from dataverse.datasets.base import BaseDataset + + +try: + import torch +except ImportError: + torch = None + +logger = logging.getLogger("dataverse") +logger.setLevel(logging.INFO) +logger.addHandler(RichHandler(markup=True, rich_tracebacks=True, log_time_format="[%m-%d %H:%M:%S]")) + + +def dataset_from_config(config: DictConfig, **additional_params) -> BaseDataset: + """ + Instantiates a dataset from a config. + """ + assert "target" in config, "Expected key `target` to be the class name under dataverse.datasets." + + target = config.target + params = config.get("params", dict()) + params.update(additional_params) + + module_name, cls_name = target.rsplit(".", 1) + + try: + dataset_module = importlib.import_module(f"dataverse.datasets.{module_name}", package=None) + except ImportError: + raise ImportError( + f"Failed to import module due to the following reasons, " + f"which can be potentially resolved by: `pip install dataverse[{module_name}]`." + ) + + assert hasattr(dataset_module, cls_name), f"Class {cls_name} not found in module {module_name}." + dataset_inst = getattr(dataset_module, cls_name)(**params) + + assert isinstance(dataset_inst, BaseDataset), f"{cls_name} is not a BaseDataset." + + return dataset_inst + + +def print_rich_dict(data_dict): + """ + Convert a dictionary into a rich Table with columns: + Key, Value Type, Value Shape, Device, Mean, Min, and Max (if value is a numpy or pytorch array). + + Args: + data_dict (dict): Dictionary to convert. + """ + # Create a Rich Table + table = Table(title="Dictionary Details") + + # Define the columns + table.add_column("Key", justify="left", style="cyan", no_wrap=True) + table.add_column("Value Type", justify="left", style="magenta") + table.add_column("Value Shape", justify="left", style="green") + table.add_column("Device", justify="left", style="blue") + table.add_column("Mean", justify="right", style="yellow") + table.add_column("Min", justify="right", style="red") + table.add_column("Max", justify="right", style="red") + + # Iterate through the dictionary + for key, value in data_dict.items(): + value_type = str(type(value)).lstrip("").replace("torch.", "").strip("'") + device = "-" + mean = min_val = max_val = "-" + + # Check if value is a NumPy array + if isinstance(value, np.ndarray): + shape = str(value.shape) + mean = f"{value.mean():.4f}" + min_val = f"{value.min():.4f}" + max_val = f"{value.max():.4f}" + + # Check if value is a PyTorch tensor + elif torch is not None and isinstance(value, torch.Tensor): + shape = str(tuple(value.shape)) + device = str(value.device) + mean = f"{value.float().mean().item():.4f}" + min_val = f"{value.min().item():.4f}" + max_val = f"{value.max().item():.4f}" + + else: + # For other data types + shape = str(np.shape(value)) + + # Add row to the table + table.add_row(str(key), value_type, shape, device, mean, min_val, max_val) + + # Print the table + console = Console() + console.print(table) diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/av_metadata.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/av_metadata.py new file mode 100644 index 00000000..e275b20f --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/av_metadata.py @@ -0,0 +1,491 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import json +import os +import tarfile + +import numpy as np +import yaml +from pyquaternion import Quaternion +from scipy.interpolate import interp1d + +from .camera_model import FThetaCamera +from .rig import get_rig_transform + + +def get_egopose_interp(egoposes): + """We move the egoposes to the coordinate frame defined by the + first egopose. Then we return an interpolator for the ego pose + in that frame. + """ + quats = [Quaternion(x=row["qx"], y=row["qy"], z=row["qz"], w=row["qw"]) for row in egoposes] + xyzs = np.array([[row["x"], row["y"], row["z"]] for row in egoposes]) + + # choose the first timestep to define the global coordinate frame + TIX = 0 + qbase = quats[TIX].inverse + qbase_rot = qbase.rotation_matrix + xyzbase = -qbase_rot @ xyzs[TIX] + cliptbase = egoposes[TIX]["t"] + + # transform to the base frame + quats = [qbase * quat for quat in quats] + xyzs = (qbase_rot @ xyzs.T + xyzbase.reshape((3, 1))).T + + # now adjust the timestamps to the base timestamp + clipts = np.array([(row["t"] - cliptbase) * 1e-6 for row in egoposes]) + + # ready for the interpolator + states = [{"quat": quat, "xyz": xyz, "t": t} for quat, xyz, t in zip(quats, xyzs, clipts)] + interp = get_6dof_interpolator(states) + + return interp, cliptbase + + +def get_6dof_interpolator(states): + """states is a list of timestamped 6 DOF poses structured like + [ + { + quat: Quaternion, + xyz: [x,y,z], + t: t, + dims: [length,width,height], (optional) + }, + ] + Before feeding to the interpolator, we sort by time, normalize the quaternions, + canonicalize the quaternion directions, and cast to float32. + The interpolator returns (qw, qx, qy, qz, x, y, z) + """ + # sort by time + states = sorted(states, key=lambda x: x["t"]) + + # normalize quaternions + # https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L847 + quats = [state["quat"] for state in states] + [quat._fast_normalise() for quat in quats] + quats = steady_quaternion_direction(quats) + + interpvals = np.array( + [ + [ + quat.w, + quat.x, + quat.y, + quat.z, + state["xyz"][0], + state["xyz"][1], + state["xyz"][2], + ] + for quat, state in zip(quats, states) + ], + dtype=np.float32, + ) + + # also interpolate bbox dimensions if available + if "dims" in states[0]: + interpvals = np.concatenate( + ( + interpvals, + np.array([state["dims"] for state in states], dtype=np.float32), + ), + 1, + ) + + tvals = np.array([state["t"] for state in states], dtype=np.float32) + # make sure t parameter is strictly increasing + assert np.all(0 < tvals[1:] - tvals[:-1]), tvals + + interp = { + "interp": interp1d( + tvals, + interpvals, + kind="linear", + axis=0, + copy=False, + bounds_error=True, + assume_sorted=True, + ), + "tmin": tvals[0], + "tmax": tvals[-1], + } + + return interp + + +def steady_quaternion_direction(quats): + """negate any of the quaternions as necessary + to keep the orientation consistent + https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L847 + """ + newquats = [quats[0]] + for q1 in quats[1:]: + q0 = newquats[-1] + dot = np.dot(q0.q, q1.q) + if dot >= 0.0: + newquats.append(q1) + else: + newquats.append(-q1) + return newquats + + +def get_obstacle_interp(og_obstacles, egopose_lerp, cliptbase): + tvals, obstacles = local_time_filter(og_obstacles, egopose_lerp["tmin"], egopose_lerp["tmax"], cliptbase) + + # evaluate ego pose at the obstacle timestamps + egoposes = egopose_lerp["interp"](tvals) + + # move obstacles into those coordinate frames + obsquats, obsxyzs = convert_full_obstacle_state(obstacles, egoposes) + + # track id -> 6 DOF states + # TODO it seems sometimes tracks have no detections for ~1.5 seconds, + # linear interpolation is probably bad at those time scales. + processed = {} + for row, tval, obsquat, obsxyz in zip(obstacles, tvals, obsquats, obsxyzs): + if not row["id"] in processed: + processed[row["id"]] = [] + processed[row["id"]].append( + { + "quat": obsquat, + "xyz": obsxyz, + "t": tval, + "dims": [row["le"], row["wi"], row["he"]], + } + ) + + # filter any tracks with fewer than 2 observations since otherwise we can't interp + processed = {k: v for k, v in processed.items() if 1 < len(v)} + + # interpolators + interps = {k: get_6dof_interpolator(v) for k, v in processed.items()} + + return interps + + +def local_time_filter(obstacles, tmin, tmax, cliptbase): + """Evaluate the time for each of the detections relative to the base time, + and remove any detections from outside the time window when + we have the ego pose + """ + # convert obstacle timestamps to the clip time frame + tvals = np.array([(row["t"] - cliptbase) * 1e-6 for row in obstacles], dtype=np.float32) + + # filter out any obstacles outside of the range when we have ego pose + obsmask = np.logical_and(tmin <= tvals, tvals <= tmax) + tvals = tvals[obsmask] + obstacles = [row for rowi, row in enumerate(obstacles) if obsmask[rowi]] + + return tvals, obstacles + + +def convert_full_obstacle_state(obstacles, egoposes): + # catch the case where there are no detections quickly + if len(obstacles) == 0: + return [], np.empty((0, 3), dtype=np.float32) + + # Nx3 ego xyzs and N list of ego quaternions + egoquats = [] + egoxyzs = [] + for qw, qx, qy, qz, x, y, z in egoposes: + egoquats.append(Quaternion(w=qw, x=qx, y=qy, z=qz)) + egoxyzs.append(np.array([x, y, z])) + egorots = np.stack([quat.rotation_matrix for quat in egoquats]).astype(np.float32) + egoxyzs = np.stack(egoxyzs) + + obsquats = [] + obsxyzs = [] + for rowi, row in enumerate(obstacles): + obsxyzs.append([row["x"], row["y"], row["z"]]) + # only yaw reported in the detections + obsquats.append(Quaternion(axis=[0, 0, 1], angle=row["theta"])) + obsxyzs = np.array(obsxyzs, dtype=np.float32) + + # transform + obsquats = [egoquat * obsquat for egoquat, obsquat in zip(egoquats, obsquats)] + obsxyzs = np.squeeze(egorots @ obsxyzs[:, :, np.newaxis], axis=2) + egoxyzs + + return obsquats, obsxyzs + + +def pose_to_corn(qw, qx, qy, qz, x, y, z, le, wi, he): + quat = Quaternion(w=qw, x=qx, y=qy, z=qz) + + le2 = le / 2 + wi2 = wi / 2 + he2 = he / 2 + + corn = np.array( + [ + [-le2, -wi2, -he2], + [le2, -wi2, -he2], + [le2, wi2, -he2], + [-le2, wi2, -he2], + [-le2, -wi2, he2], + [le2, -wi2, he2], + [le2, wi2, he2], + [-le2, wi2, he2], + ] + ) + + corn = corn @ quat.rotation_matrix.T + np.array([x, y, z]) + + return corn + + +def get_calibration_tfs(calibration, lidarkey, camkeys): + tfs = {} + tfs["lidar2rig"] = get_rig_transform(calibration[lidarkey], rig2sensor=False) + + for camkey in camkeys: + tfs[f"rig2{camkey}"] = get_rig_transform(calibration[camkey], rig2sensor=True) + tfs[f"ftheta{camkey}"] = FThetaCamera.from_dict(calibration[camkey]) + + # make sure didn't overwrite anything + assert len(tfs) == 1 + 2 * len(camkeys), len(tfs) + + return tfs + + +def get_clip_to_tar(datapath): + """Read the clip_to_tar.json metadata for the folder datapath. Returns + a dictionary + id -> (tar file location, key for this clip in that tar file) + """ + # clipid->tar stored at clip_to_tar.json + fpath = os.path.join(datapath, "clip_to_tar.json") + + # print("reading clip_to_tar.json from", fpath) + with open(fpath, "r") as reader: + info = json.load(reader) + + # id -> (tar file location, key for this clip in that tar file) + parsed_info = {key2id(key): (os.path.join(datapath, val), key) for key, val in info.items()} + + # shouldn't lose any clips assuming the id is unique + assert len(parsed_info) == len(info), f"{len(parsed_info)} {len(info)}" + + return parsed_info + + +def key2id(key): + """The clip ids are saved as e.g. '6c896d92-ad90-11ed-ac93-00044bf65f70_2-clip396904' + but the "_2" is not the same across maglev jobs. So we remove that part to + get the clip id. + """ + sessionid, s1 = key.split("_") + _, clipid = s1.split("-") + return (sessionid, clipid) + + +def parse_obstacle_data(info, check_sensor): + """Based on https://tegra-sw-opengrok.nvidia.com/source/xref/ndas-main/tools/avmf/tools/scripts/sql_gt_to_rclog/av_proto_makers.py#183 + + "info" is a list of detections. We collect the detections for each track id + to get a dictionary track id -> [detection1, detection2,...] with detections + sorted by time. + + We also parse the sensor name corresponding to the coordinate frame the boxes + are represented in. + """ + processed = [] + duplicate_tracker = {} + for row in info["labels_data"]: + row = {k: v for k, v in zip(info["labels_keys"], row)} + label = json.loads(row["label_data"]) + + # skip these + if "emptyLabel" in label and label["emptyLabel"]: + continue + + atts = {att["name"]: att for att in label["shape3d"]["attributes"]} + # make sure no duplicates in the attribute names + assert len(atts) == len(label["shape3d"]["attributes"]) + + id = int(float(atts["gt_trackline_id"]["text"])) + # skip these + if id is None or id < 0: + continue + + # TODO seems we sometimes (~1% of clips) have duplicates. This + # fixes it by taking only one detection per track per lidar spin. + moment_id = (id, row["frame_number"]) + if moment_id in duplicate_tracker: + continue + duplicate_tracker[moment_id] = True + + processed.append( + { + "x": label["shape3d"]["cuboid3d"]["center"]["x"], + "y": label["shape3d"]["cuboid3d"]["center"]["y"], + "z": label["shape3d"]["cuboid3d"]["center"]["z"], + "theta": np.arctan2(atts["direction"]["vec2"]["y"], atts["direction"]["vec2"]["x"]), + "le": label["shape3d"]["cuboid3d"]["dimensions"]["x"], + "wi": label["shape3d"]["cuboid3d"]["dimensions"]["y"], + "he": label["shape3d"]["cuboid3d"]["dimensions"]["z"], + "npct": atts["label_name"]["enum"], + "t": int(float(atts["timestamp"]["text"])), + "id": id, + } + ) + + # make sure sensor coordinate frame is the same as ego pose + assert row["sensor_name"] == check_sensor, f"{row['sensor_name']} {check_sensor}" + + return processed + + +def parse_egopose_data(egopose_info): + # TODO verify that all timestamps are on the same clock as the detections + # and videos + parsed = [] + sensor_names = [] + for row in egopose_info: + assert row["coordinate_frame"] == row["sensor_name"], f"{row['coordinate_frame']} {row['sensor_name']}" + sensor_names.append(row["coordinate_frame"]) + parsed.append( + { + "x": row["x"], + "y": row["y"], + "z": row["z"], + "qx": row["qx"], + "qy": row["qy"], + "qz": row["qz"], + "qw": row["qw"], + "t": row["timestamp"], + } + ) + # sort chronologically + parsed = sorted(parsed, key=lambda x: x["t"]) + + # make sure all poses are in the same sensor frame + assert len(set(sensor_names)) == 1, set(sensor_names) + sensor_name = sensor_names[0] + + return parsed, sensor_name + + +def get_video_parameters(datapaths, camkeys): + """Load the pre-processing config file for the dataset. + Currently we assert that all cameras use the same parameters, + we could loosen that in the future. + """ + collect = {"crop_h": [], "crop_w": [], "size_h": [], "size_w": [], "fps": []} + for datapath in datapaths: + f = os.path.join(datapath, "generator_config.yaml") + + # print("reading", f) + with open(f, "r") as reader: + conf = yaml.safe_load(reader) + + for camkey in camkeys: + for key in collect: + collect[key].append(conf["data"][camkey.replace(".mp4", "")][key]) + # make sure they're all the same + for key in collect: + assert len(set(collect[key])) == 1, collect[key] + + # select the first one since they're all the same + out = {k: v[0] for k, v in collect.items()} + + return out + + +def adjust_calib_name(name): + """The sensor names in the calibration file are e.g. camera:cross:left:120fov. + We adjust them to match the names in the wds tar files, eg camera_cross_left_120fov.mp4 + """ + # the sensor kind (eg radar/camera/lidar etc) + kind = name.split(":")[0] + + if kind == "camera": + return f"{name.replace(':', '_')}.mp4" + return name.replace(":", "_") + + +def parse_calibration_data(calibration): + # sensor calibration dictionary + rig_info = {adjust_calib_name(sensor["name"]): sensor for sensor in calibration["rig"]["sensors"]} + # make sure no overwriting when adjusting the sensor name + assert len(rig_info) == len(calibration["rig"]["sensors"]) + + # TODO seems we can get adjustment of rig coordinate frame to ego bbox? + # from calibration['rig']['vehicle']['value']['body']['boundingBoxPosition'] + + egoparams = { + "le": calibration["rig"]["vehicle"]["value"]["body"]["length"], + "wi": calibration["rig"]["vehicle"]["value"]["body"]["width"], + "he": calibration["rig"]["vehicle"]["value"]["body"]["height"], + "platform": calibration["rig"]["properties"]["platform_name"], + } + + return rig_info, egoparams + + +def extract_obstacle_and_epomotion_from_tar(sample_key, camera_keys, clip2tar, load_obstacle=True): + """Extract egomotions from tar, used by AV1.1""" + + metadata = {} + try: + cid = key2id(sample_key) + if not cid in clip2tar: + return metadata + + tarf, tarkey = clip2tar[cid] + tarreader = tarfile.open(tarf) + for camera_name in camera_keys: + camera_metadata = {} + camera = camera_name.replace(".mp4", "") + egopose_file = tarreader.extractfile(f"{tarkey}.{camera}_egomotion.json") + egopose_info = json.loads(egopose_file.read()) + egopose_parsed, sensor_name = parse_egopose_data(egopose_info) + camera_metadata["egopose"] = egopose_parsed + camera_metadata["sensor_name"] = sensor_name + + # obstacle data + if load_obstacle: + obstacle_file = tarreader.extractfile(f"{tarkey}.obstacle_3d.npz") + obstacle_info = np.load(obstacle_file, allow_pickle=True) + obstacle_parsed = parse_obstacle_data(obstacle_info, check_sensor=sensor_name) + camera_metadata["obstacles"] = obstacle_parsed + + metadata[camera_name] = camera_metadata + except Exception as error: + print(f"Failed to extract egomotion metadata for {sample_key}: {error}") + return None + return metadata + + +def extract_calibration_from_tar(sample_key, clip2tar): + """Extract calibrations from tar, used by AV1.1""" + + metadata = {} + try: + cid = key2id(sample_key) + if not cid in clip2tar: + return metadata + + tarf, tarkey = clip2tar[cid] + tarreader = tarfile.open(tarf) + calibration_file = tarreader.extractfile(f"{tarkey}.rig.json") + calibration_info = json.loads(calibration_file.read()) + rig_info, egoparams = parse_calibration_data(calibration_info) + metadata["rig_info"] = rig_info + metadata["egoparams"] = egoparams + except Exception as error: + print(f"Failed to extract calibration for {sample_key}: {error}") + return None + return metadata diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/camera_model.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/camera_model.py new file mode 100644 index 00000000..3dcc2f8a --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/camera_model.py @@ -0,0 +1,665 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Camera model definitions.""" + +import json +import math +from typing import Any, Dict, Tuple, TypeVar, Union + +import numpy as np +import torch +from numpy.polynomial.polynomial import Polynomial +from scipy.optimize import curve_fit + + +CropParams = TypeVar("CropParams") +ScaleParams = TypeVar("ScaleParams") + + +class IdealPinholeCamera: + """Reperesents an ideal pinhole camera with no distortions. + + You can either pass in the fov or you can pass in the actual + focal point parameters. It is the users choice. If you pass + in the fov, then the f_x, f_y parameters are computed for you. + Otherwise, they are directly inserted into the intrinsic matrix. + + """ + + def __init__( + self, + fov_x_deg: Union[float, int] = None, + fov_y_deg: Union[float, int] = None, + f_x: Union[float, int] = None, + f_y: Union[float, int] = None, + width: int = 3848, + height: int = 2168, + ): + """The __init__ function. + + Args: + fov_x_deg (Union[float, int]): the horizontal FOV in degrees. + fov_y_deg (Union[float, int]): the vertical FOV in degrees. + f_x (Union[float, int]): the f_x value of the intrinsic calibration + matrix + f_y (Union[float, int]): the f_y value of the intrinsic calibration + matrix + width (int): the width of the image. Defaults to 3848 + height (int): the height of the image. Defaults to 2168 + """ + + if f_x and fov_x_deg or f_y and fov_y_deg: + raise ValueError( + "Either f_x,f_y or fov_x_deg, fov_y_deg can" + "be passed in but not both. User must select which" + "operational mode you intend to use. If you want to" + "directly insert fx,fy into the intrinsic calibration" + "matrix then do not pass in fov_x_deg or fov_y_deg" + "and if you want to compute f_x, f_y from the FOV then" + "do not pass in f_x, f_y" + ) + + self._width = width + self._height = height + self._cx = width / 2 + self._cy = height / 2 + + # You can pass in the values directly. + if f_x and f_y: + self._f_x = f_x + self._f_y = f_y + else: + self._focal_from_fov(fov_x_deg, fov_y_deg) + + # The intrinsics matrix + self._k = np.asarray( + [[self._f_x, 0, self._cx], [0, self._f_y, self._cy], [0, 0, 1]], + dtype=np.float32, + ) + # The inverse of the intrinsics matrix (for backprojection) + self._k_inv = np.asarray( + [ + [1.0 / self._f_x, 0, -self._cx / self._f_x], + [0, 1.0 / self._f_y, -self._cy / self._f_y], + [0, 0, 1], + ], + dtype=np.float32, + ) + + @property + def width(self) -> int: + """Returns the width of the sensor.""" + return self._width + + @property + def height(self) -> int: + """Returns the height of the sensor.""" + return self._height + + def _focal_from_fov(self, fov_x_deg: Union[float, int], fov_y_deg: Union[float, int]): + """Compute the focal length from horizontal and vertical FOVs. + + Args: + fov_x_deg (Union[float, int]): the horizontal FOV in degrees. + fov_y_deg (Union[float, int]): the vertical FOV in degrees. + """ + fov_x = np.radians(fov_x_deg) + fov_y = np.radians(fov_y_deg) + self._f_x = self._width / (2.0 * np.tan(fov_x * 0.5)) + self._f_y = self._height / (2.0 * np.tan(fov_y * 0.5)) + + def ray2pixel(self, rays: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Project 3D rays to 2D pixel coordinates. + + Args: + rays (np.ndarray): the rays as (N, 3) where N corresponds to + the number of rays and 3 is the (x,y,z) coordinates for each + ray. + + Returns: + projected (np.ndarray): Shape (N,2) the projected pixel coordinates + where N is the number of points and 2 corresponds to the (x,y) dimensions. + valid (np.ndarray): of Shape (N,) the validity flag for each projected pixel. + Valid is a boolean array that can be used for indexing rays + that are within FOV. + + """ + if np.ndim(rays) == 1: + rays = rays[np.newaxis, :] + + rays = rays.astype(np.float32) + + r = rays / rays[:, 2:] + + projected = np.matmul(self._k, r.T).T + + x_ok = np.logical_and(0 <= projected[:, 0], projected[:, 0] < self._width) + y_ok = np.logical_and(0 <= projected[:, 1], projected[:, 1] < self._height) + valid = np.logical_and(x_ok, y_ok) + return projected[:, :2], valid + + def pixel2ray(self, pixels: np.ndarray) -> np.ndarray: + """Backproject 2D pixels into 3D rays. + + Args: + pixels (np.ndarray): the pixels to backproject. Size of (n_points, 2), where the first + column contains the `x` values, and the second column contains the `y` values. + + Returns: + rays (np.ndarray): the backprojected 3D rays. + """ + if np.ndim(pixels) == 1: + pixels = pixels[np.newaxis, :] + + pixels = pixels.astype(np.float32) + + # Add the third component of ones + pixels = np.c_[pixels, np.ones((pixels.shape[0], 1), dtype=np.float32)] + rays = np.matmul(self._k_inv, pixels.T).T + + # Normalize the rays + norm = np.linalg.norm(rays, axis=1, keepdims=True) + norm[norm == 0] = 1 + return rays / norm + + +class FThetaCamera: + """Defines an FTheta camera model.""" + + @classmethod + def from_rig(cls, rig_file: str, sensor_name: str): + """Helper method to initialize a new object using a rig file and the sensor's name. + + Args: + rig_file (str): the rig file path. + sensor_name (str): the name of the sensor. + + Returns: + FThetaCamera: the newly created object. + """ + with open(rig_file, "r") as fp: + rig = json.load(fp) + + # Parse the properties from the rig file + sensors = rig["rig"]["sensors"] + sensor = None + sensor_found = False + + for sensor in sensors: + if sensor["name"] == sensor_name: + sensor_found = True + break + + if not sensor_found: + raise ValueError(f"The camera '{sensor_name}' was not found in the rig!") + + return cls.from_dict(sensor) + + @classmethod + def from_dict(cls, rig_dict: Dict[str, Any]): + """Helper method to initialize a new object using a dictionary of the rig. + + Args: + rig_dict (dict): the sensor dictionary to initialize with. + + Returns: + FThetaCamera: the newly created object. + """ + cx, cy, width, height, bw_poly = FThetaCamera.get_ftheta_parameters_from_json(rig_dict) + return cls(cx, cy, width, height, bw_poly) + + @classmethod + def from_intrinsics_array(cls, intrinsics: np.ndarray): + """Helper method to initialize a new object using an array of intrinsics. + + Args: + intrinsics (np.ndarray): the intrinsics array. The ordering is expected to be + "cx, cy, width, height, bw_poly". This is the same ordering as the `intrinsics` + property of this class. + + Returns: + FThetaCaamera: the newly created object. + """ + return cls( + cx=intrinsics[0], + cy=intrinsics[1], + width=intrinsics[2], + height=intrinsics[3], + bw_poly=intrinsics[4:], + ) + + def __init__(self, cx: float, cy: float, width: int, height: int, bw_poly: np.ndarray): + """The __init__ method. + + Args: + cx (float): optical center x. + cy (float): optical center y. + width (int): the width of the image. + height (int): the height of the image. + bw_poly (np.ndarray): the backward polynomial of the FTheta model. + """ + self._center = np.asarray([cx, cy], dtype=np.float32) + self._width = int(width) + self._height = int(height) + self._bw_poly = Polynomial(bw_poly) + self._fw_poly = self._compute_fw_poly() + # Other properties that need to be computed + self._horizontal_fov = None + self._vertical_fov = None + self._max_angle = None + self._max_ray_angle = None + # Populate the array of intrinsics + self._intrinsics = np.append([cx, cy, width, height], bw_poly).astype(np.float32) + + self._update_calibrated_camera() + + @staticmethod + def get_ftheta_parameters_from_json(rig_dict: Dict[str, Any]) -> Tuple[Any]: + """Helper method for obtaining FTheta camera model parameters from a rig dict. + + Args: + rig_dict (Dict[str, Any]): the rig dictionary to parse. + + Raises: + ValueError: if the provided rig is not supported. + AssertionError: if the provided model is supported, but cannot be parsed properly. + + Returns: + Tuple[Any]: the values `cx`, `cy`, `width`, `height` and `bw_poly` that were parsed. + """ + props = rig_dict["properties"] + + if props["Model"] != "ftheta": + raise ValueError("The given camera is not an FTheta camera") + + cx = float(props["cx"]) + cy = float(props["cy"]) + width = int(props["width"]) + height = int(props["height"]) + + if "bw-poly" in props: # Is this a regular rig? + poly = props["bw-poly"] + elif "polynomial" in props: # Is this a VT rig? + # VT rigs have a slightly different format, so need to handle these + # specifically. Refer to the following thread for more details: + # https://nvidia.slack.com/archives/C017LLEG763/p1633304770105300 + poly_type = props["polynomial-type"] + assert poly_type == "pixeldistance-to-angle", ( + "Encountered an unsupported VT rig. Only `pixeldistance-to-angle` " + f"polynomials are supported (got {poly_type}). Rig:\n{rig_dict}" + ) + + linear_c = float(props["linear-c"]) if "linear-c" in props else None + linear_d = float(props["linear-d"]) if "linear-d" in props else None + linear_e = float(props["linear-e"]) if "linear-e" in props else None + + # If we had all the terms present, sanity check to make sure they are [1, 0, 0] + if linear_c is not None and linear_d is not None and linear_e is not None: + assert linear_c == 1.0, f"Expected `linear-c` term to be 1.0 (got {linear_c}. Rig:\n{rig_dict})" + assert linear_d == 0.0, f"Expected `linear-d` term to be 0.0 (got {linear_d}. Rig:\n{rig_dict})" + assert linear_e == 0.0, f"Expected `linear-e` term to be 0.0 (got {linear_e}. Rig:\n{rig_dict})" + + # If we're here, then it means we can parse the rig successfully. + poly = props["polynomial"] + else: + raise ValueError(f"Unable to parse the rig. Only FTheta rigs are supported! Rig:\n{rig_dict}") + + bw_poly = [np.float32(val) for val in poly.split()] + return cx, cy, width, height, bw_poly + + @property + def fov(self) -> tuple: + """Returns a tuple of horizontal and vertical fov of the sensor.""" + if self._vertical_fov is None or self._horizontal_fov is None: + self._compute_fov() + return self._horizontal_fov, self._vertical_fov + + @property + def width(self) -> int: + """Returns the width of the sensor.""" + return self._width + + @property + def height(self) -> int: + """Returns the height of the sensor.""" + return self._height + + @property + def center(self) -> np.ndarray: + """Returns the center of the sensor.""" + return self._center + + @property + def intrinsics(self) -> np.ndarray: + """Obtain an array of the intrinsics of this camera model. + + Returns: + np.ndarray: an array of intrinsics. The ordering is "cx, cy, width, height, bw_poly". + dtype is np.float32. + """ + return self._intrinsics + + def __str__(self): + """Returns a string representation of this object.""" + return ( + f"FTheta camera model:\n\t{self._bw_poly}\n\t" + f"center={self._center}\n\twidth={self._width}\n\theight={self._height}\n\t" + f"h_fov={np.degrees(self._horizontal_fov)}\n\tv_fov={np.degrees(self._vertical_fov)}" + ) + + def _update_calibrated_camera(self): + """Updates the internals of this object after calulating various properties.""" + self._compute_fov() + self._max_ray_angle = (self._max_angle).copy() + is_fw_poly_slope_negative_in_domain = False + ray_angle = (np.float32(self._max_ray_angle)).copy() + deg2rad = np.pi / 180.0 + while ray_angle >= np.float32(0.0): + temp_dval = self._fw_poly.deriv()(self._max_ray_angle).item() + if temp_dval < 0: + is_fw_poly_slope_negative_in_domain = True + ray_angle -= deg2rad * np.float32(1.0) + + if is_fw_poly_slope_negative_in_domain: + ray_angle = (np.float32(self._max_ray_angle)).copy() + while ray_angle >= np.float32(0.0): + ray_angle -= deg2rad * np.float32(1.0) + raise ArithmeticError("FThetaCamera: derivative of distortion within image interior is negative") + + # Evaluate the forward polynomial at point (self._max_ray_angle, 0) + # Also evaluate its derivative at the same point + val = self._fw_poly(self._max_ray_angle).item() + dval = self._fw_poly.deriv()(self._max_ray_angle).item() + + if dval < 0: + raise ArithmeticError("FThetaCamera: derivative of distortion at edge of image is negative") + + self._max_ray_distortion = np.asarray([val, dval], dtype=np.float32) + + def _compute_fw_poly(self): + """Computes the forward polynomial for this camera. + + This function is a replication of the logic in the following file from the DW repo: + src/dw/calibration/cameramodel/CameraModels.cpp + """ + + def get_max_value(p0, p1): + return np.linalg.norm(np.asarray([p0, p1], dtype=self._center.dtype) - self._center) + + max_value = 0.0 + + size = (self._width, self._height) + value = get_max_value(0.0, 0.0) + max_value = max(max_value, value) + value = get_max_value(0.0, size[1]) + max_value = max(max_value, value) + value = get_max_value(size[0], 0.0) + max_value = max(max_value, value) + value = get_max_value(size[0], size[1]) + max_value = max(max_value, value) + + SAMPLE_COUNT = 500 + samples_x = [] + samples_b = [] + step = max_value / SAMPLE_COUNT + x = step + + for _ in range(0, SAMPLE_COUNT): + p = np.asarray([self._center[0] + x, self._center[1]], dtype=np.float32) + ray, _ = self.pixel2ray(p) + xy_norm = np.linalg.norm(ray[0, :2]) + theta = np.arctan2(float(xy_norm), float(ray[0, 2])) + samples_x.append(theta) + samples_b.append(float(x)) + x += step + + x = np.asarray(samples_x, dtype=np.float64) + y = np.asarray(samples_b, dtype=np.float64) + # Fit a 4th degree polynomial. The polynomial function is as follows: + + def f(x, b, x1, x2, x3, x4): + """4th degree polynomial.""" + return b + x * (x1 + x * (x2 + x * (x3 + x * x4))) + + # The constant in the polynomial should be zero, so add the `bounds` condition. + # FIXME(mmaghoumi) DW mentions disabling input normalization, what's that?? + # - the compuation is more stable if the data is normalized before the fitting process. + coeffs, _ = curve_fit( + f, + x, + y, + bounds=( + [0, -np.inf, -np.inf, -np.inf, -np.inf], + [np.finfo(np.float64).eps, np.inf, np.inf, np.inf, np.inf], + ), + ) + # Return the polynomial and hardcode the bias value to 0 + return Polynomial([np.float32(val) if i > 0 else 0 for i, val in enumerate(coeffs)]) + + def pixel2ray(self, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Backproject 2D pixels into 3D rays. + + Args: + x (np.ndarray): the pixels to backproject. Size of (n_points, 2), where the first + column contains the `x` values, and the second column contains the `y` values. + + Returns: + rays (np.ndarray): the backprojected 3D rays. Size of (n_points, 3). + valid (np.ndarray): bool flag indicating the validity of each backprojected pixel. + """ + # Make sure x is n x 2 + if np.ndim(x) == 1: + x = x[np.newaxis, :] + + # Fix the type + x = x.astype(np.float32) + xd = x - self._center + xd_norm = np.linalg.norm(xd, axis=1, keepdims=True) + alpha = self._bw_poly(xd_norm) + sin_alpha = np.sin(alpha) + + rx = sin_alpha * xd[:, 0:1] / xd_norm + ry = sin_alpha * xd[:, 1:] / xd_norm + rz = np.cos(alpha) + + rays = np.hstack((rx, ry, rz)) + # special case: ray is perpendicular to image plane normal + valid = (xd_norm > np.finfo(np.float32).eps).squeeze() + rays[~valid, :] = (0, 0, 1) # This is what DW sets these rays to + + # note: + # if constant coefficient of bwPoly is non-zero, + # the resulting ray might not be normalized. + return rays, valid + + def ray2pixel(self, rays: np.ndarray) -> np.ndarray: + """Project 3D rays to 2D pixel coordinates. + + Args: + rays (np.ndarray): the rays. + + Returns: + result (np.ndarray): the projected pixel coordinates. + """ + # Make sure the input shape is (n_points, 3) + if np.ndim(rays) == 1: + rays = rays[np.newaxis, :] + + # Fix the type + rays = rays.astype(np.float32) + # TODO(restes) combine 2 and 3 column norm for rays? + xy_norm = np.linalg.norm(rays[:, :2], axis=1, keepdims=True) + cos_alpha = rays[:, 2:] / np.linalg.norm(rays, axis=1, keepdims=True) + + alpha = np.empty_like(cos_alpha) + cos_alpha_condition = np.logical_and(cos_alpha > np.float32(-1.0), cos_alpha < np.float32(1.0)).squeeze() + alpha[cos_alpha_condition] = np.arccos(cos_alpha[cos_alpha_condition]) + alpha[~cos_alpha_condition] = xy_norm[~cos_alpha_condition] + + delta = np.empty_like(cos_alpha) + alpha_cond = alpha <= self._max_ray_angle + delta[alpha_cond] = self._fw_poly(alpha[alpha_cond]) + # For outside the model (which need to do linear extrapolation) + delta[~alpha_cond] = ( + self._max_ray_distortion[0] + (alpha[~alpha_cond] - self._max_ray_angle) * self._max_ray_distortion[1] + ) + + # Determine the bad points with a norm of zero, and avoid division by zero + bad_norm = xy_norm <= 0 + xy_norm[bad_norm] = 1 + delta[bad_norm] = 0 + # compute pixel relative to center + scale = delta / xy_norm + pixel = scale * rays + + # Handle the edge cases (ray along image plane normal) + edge_case_cond = (xy_norm <= np.float32(0.0)).squeeze() + pixel[edge_case_cond, :] = rays[edge_case_cond, :] + result = pixel[:, :2] + self._center + return result + + def _get_pixel_fov(self, pt: np.ndarray) -> float: + """Gets the FOV for a given point. Used internally for FOV computation. + + Args: + pt (np.ndarray): 2D pixel. + + Returns: + fov (float): the FOV of the pixel. + """ + ray, _ = self.pixel2ray(pt) + fov = np.arctan2(np.linalg.norm(ray[:, :2], axis=1), ray[:, 2]) + return fov + + def _compute_fov(self): + """Computes the FOV of this camera model.""" + max_x = self._width - 1 + max_y = self._height - 1 + + point_left = np.asarray([0, self._center[1]], dtype=np.float32) + point_right = np.asarray([max_x, self._center[1]], dtype=np.float32) + point_top = np.asarray([self._center[0], 0], dtype=np.float32) + point_bottom = np.asarray([self._center[0], max_y], dtype=np.float32) + + fov_left = self._get_pixel_fov(point_left) + fov_right = self._get_pixel_fov(point_right) + fov_top = self._get_pixel_fov(point_top) + fov_bottom = self._get_pixel_fov(point_bottom) + + self._vertical_fov = fov_top + fov_bottom + self._horizontal_fov = fov_left + fov_right + self._compute_max_angle() + + def _compute_max_angle(self): + """Computes the maximum ray angle for this camera.""" + max_x = self._width - 1 + max_y = self._height - 1 + + p = np.asarray([[0, 0], [max_x, 0], [0, max_y], [max_x, max_y]], dtype=np.float32) + + self._max_angle = max( + max(self._get_pixel_fov(p[0, ...]), self._get_pixel_fov(p[1, ...])), + max(self._get_pixel_fov(p[2, ...]), self._get_pixel_fov(p[3, ...])), + ) + + def is_ray_inside_fov(self, ray: np.ndarray) -> bool: + """Determines whether a given ray is inside the FOV of this camera. + + Args: + ray (np.ndarray): the 3D ray. + + Returns: + bool: whether the ray is inside the FOV. + """ + if np.ndim(ray) == 1: + ray = ray[np.newaxis, :] + + ray_angle = np.arctan2(np.linalg.norm(ray[:, :2], axis=1), ray[:, 2]) + return ray_angle <= self._max_angle + + +def rays_to_pixels_batch(rays: torch.Tensor, cam_intrinsic: torch.Tensor) -> torch.Tensor: + """Project 3D rays to 2D pixel coordinates. + + Computes image projection for 3D points in an + F-theta camera. + More information about the projection is in the below link: + https://confluence.nvidia.com/display/DS/Camera+intrinsic+models + Args: + rays (torch.Tensor): 3D rays tensor with size (B, H, W, 3, N). + where N is the number of vertices. + cam_intrinsic (torch.Tensor): camera intrinsic. tensor with size 9. + In particular, the element 0 is img_cx, 1 is img_cy, + 2 is img_width, 3 is img_height, 4:9 is img_bw_poly. + + Returns: + pixels (torch.Tensor): the projected pixel coordinates. + Tensor is of size (B, H, W, 2, N). + """ + cam_intrinsic = cam_intrinsic.cpu().detach().numpy() + img_cx = cam_intrinsic[0] + img_cy = cam_intrinsic[1] + img_width = cam_intrinsic[2] + img_height = cam_intrinsic[3] + img_bw_poly = cam_intrinsic[4:] + ftheta_model = FThetaCamera(img_cx, img_cy, img_width, img_height, img_bw_poly) + # Computes the forward polynomial for this camera + fw_poly = list(ftheta_model._compute_fw_poly()) + device = rays.device + max_ray_angle = torch.tensor(ftheta_model._max_ray_angle, device=device) + max_ray_distortion = torch.tensor(ftheta_model._max_ray_distortion, device=device) + + ray_x = rays[:, :, :, 0, :] + ray_y = rays[:, :, :, 1, :] + ray_z = rays[:, :, :, 2, :] + ray_xy_norm = torch.linalg.norm(torch.stack([ray_x, ray_y], dim=3), dim=3) + # Compute the angle of ray with the optical axis + alpha = (math.pi / 2.0) - torch.atan2(ray_z, ray_xy_norm) + + delta_valid = ( + fw_poly[0] + fw_poly[1] * alpha + fw_poly[2] * alpha**2 + fw_poly[3] * alpha**3 + fw_poly[4] * alpha**4 + ) + # For outside the model (which need to do linear extrapolation) + delta_invalid = max_ray_distortion[0] + (alpha - max_ray_angle) * max_ray_distortion[1] + delta = torch.where(alpha <= max_ray_angle, delta_valid, delta_invalid) + + # Determine the bad points with a norm of zero, and avoid division by zero + delta = torch.where(ray_xy_norm > 0.0, delta, torch.tensor(0.0, device=device)) + ray_xy_norm = torch.where(ray_xy_norm > 0.0, ray_xy_norm, torch.tensor(1.0, device=device)) + + # compute pixel relative to center + scale = delta / ray_xy_norm + + pixel_x = img_cx + scale * ray_x + pixel_y = img_cy + scale * ray_y + pixels = torch.stack([pixel_x, pixel_y], dim=3) + + return pixels + + +def convert_to_2d_camera_model(xyz: np.ndarray, ft_camera: FThetaCamera) -> Tuple[np.ndarray, np.ndarray]: + """Converts a series of points in xyz to 2D canera coordinates. + + Args: + xyz: Batch of points [x, 3] in shape. + ft_camera: The camera model to use for the conversion to 2D coordinates. + + Returns: + Points in camera uv coordinates. + """ + P = np.array([[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [-1.0, 0.0, 0.0]]) + flu = P @ xyz.T + uv = ft_camera.ray2pixel(flu.T) + return uv diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/rig.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/rig.py new file mode 100644 index 00000000..a3ff27d2 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/ndas/rig.py @@ -0,0 +1,163 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoNet v2 rig helper functions.""" + +import typing + +import numpy as np +import torch +from scipy.spatial.transform import Rotation as R + + +def get_sensor_to_sensor_flu(sensor: str): + """Compute a rotation transformation matrix that rotate sensor to Front-Left-Up format. + + Args: + sensor (str): sensor name. + + Returns: + np.array: the resulting rotation matrix. + """ + if "cam" in sensor: + rot = [ + [0.0, 0.0, 1.0, 0.0], # + [-1.0, 0.0, 0.0, 0.0], # + [0.0, -1.0, 0.0, 0.0], # + [0.0, 0.0, 0.0, 1.0], # + ] + else: + rot = [ + [1.0, 0.0, 0.0, 0.0], # + [0.0, 1.0, 0.0, 0.0], # + [0.0, 0.0, 1.0, 0.0], # + [0.0, 0.0, 0.0, 1.0], # + ] + + return np.asarray(rot, dtype=np.float32) + + +def transform_from_eulers(rpy_deg: typing.Sequence[float], translation: typing.Sequence[float]): + """Create a 4x4 rigid transformation matrix given euler angles and translation. + + Args: + rpy_deg (typing.Sequence[float]): Euler angles as roll, pitch, yaw in degrees. + translation (typing.Sequence[float]): x, y, z translation. + + Returns: + np.array: the constructed transformation matrix. + """ + transform = np.eye(4) + transform[:3, :3] = R.from_euler(seq="xyz", angles=rpy_deg, degrees=True).as_matrix() + transform[:3, 3] = translation + + return transform.astype(np.float32) + + +def apply_transform( + transform: typing.Union[np.array, torch.Tensor], + pts: typing.Union[np.array, torch.Tensor], +): + """Applies the given transform to the given points in 3D space. + + Args: + transform (typing.Union[np.array, torch.Tensor]): the transform to apply. + Must be a 4x4 or 3x4 matrix. + pts (typing.Union[np.array, torch.Tensor]): the 3D points to trasform (shape (n, 3)) + + Returns: + typing.Union[np.array, torch.Tensor]: the transformed points. + """ + assert transform.shape in ( + (4, 4), + (3, 4), + ), "Only 4x4 and 3x4 transformation matrices are supported." + assert type(transform) in ( + torch.Tensor, + np.ndarray, + ), "Only torch tensors and numpy arrays are supported." + assert type(pts) in ( + torch.Tensor, + np.ndarray, + ), "Only torch tensors and numpy arrays are supported." + + if pts.ndim == 1: + if isinstance(pts, np.ndarray): # Numpy + pts_ = pts[np.newaxis, :] + else: # Torch + pts_ = pts.unsqueeze(0) + else: + pts_ = pts + + assert pts_.shape[1] == 3, "Input points must be of shape n x 3." + + # Extract R and T components + R = transform[:3, :3] + T = transform[:3, 3:] + # Apply! + result = R @ pts_.T + T + return result.T.squeeze() + + +def parse_sensor_to_rig(sensor: dict, ignore_correction_T: bool = False): + """Parses the provided rig-style transform dictionary into sensor to rig matrices. + + Args: + sensor: dictionary for a sensor read from a calibration file + ignore_correction_T (bool): if `True`, the correction translation values in the rig will + be ignored and set to zero. + + Returns: + sensor_to_rig (np.array): the 4x4 sensor to rig transformation matrix with the correction + factors applied. + nominal_sensor_to_rig (np.array) the nominal 4x4 sensor to rig transformation matrix + without the correction factors. + """ + sensor_to_FLU = get_sensor_to_sensor_flu(sensor["name"]) + + nominal_FLU_to_rig = transform_from_eulers( + sensor["nominalSensor2Rig_FLU"]["roll-pitch-yaw"], + sensor["nominalSensor2Rig_FLU"]["t"], + ) + + # Use or ignore the correction for translation + if "correction_rig_T" in sensor.keys() and not ignore_correction_T: + correction_T = sensor["correction_rig_T"] + else: + correction_T = np.zeros(3, dtype=np.float32) + + correction_transform = transform_from_eulers(sensor["correction_sensor_R_FLU"]["roll-pitch-yaw"], correction_T) + + sensor_to_rig = nominal_FLU_to_rig @ correction_transform @ sensor_to_FLU + nominal_sensor_to_rig = nominal_FLU_to_rig @ sensor_to_FLU + + return sensor_to_rig, nominal_sensor_to_rig + + +def get_rig_transform(sensor: dict, rig2sensor: bool): + """Obtain the camera to rig coordinate transform. + + Args: + sensor: dictionary for a sensor read from a calibration file + rig2sensor: When True, returns the rig to sensor transform, returns sensor to rig otherwise. + """ + # Calculate: + # nominal_FLU_to_rig @ correction_transform @ sensor_to_FLU + tf, _ = parse_sensor_to_rig(sensor) + FLU_to_sensor = np.linalg.inv(get_sensor_to_sensor_flu(sensor["name"])) + tf = tf @ FLU_to_sensor + + if rig2sensor: + tf = np.linalg.inv(tf) + return tf diff --git a/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/version.py b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/version.py new file mode 100644 index 00000000..44d43111 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse/dataverse/utils/version.py @@ -0,0 +1,74 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + + +VERSION_MAJOR = 1 +VERSION_MINOR = 0 +VERSION_PATCH = 0 + + +def get_git_commit_sha_short(): + """ + Returns the short SHA-1 hash of the current commit, or 'unknown' if not in a git repository. + """ + try: + # Run git command to get the short commit hash (7 characters) + commit_sha = ( + subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], stderr=subprocess.DEVNULL) + .decode("utf-8") + .strip() + ) + except (subprocess.CalledProcessError, FileNotFoundError): + commit_sha = "unknown" + return commit_sha + + +def is_git_tree_dirty(): + """ + Returns whether the git tree is dirty (has uncommitted changes). + """ + try: + # Check for uncommitted changes + dirty = subprocess.call(["git", "diff", "--quiet"], stderr=subprocess.DEVNULL) + return dirty != 0 + except (subprocess.CalledProcessError, FileNotFoundError): + return False + + +def get_version(): + """ + Returns the current version of the module in the format: + VERSION_MAJOR.VERSION_MINOR.VERSION_PATCH-GIT_COMMIT_SHA_SHORT[+GIT_TREE_DIRTY] + """ + # Get the short commit hash + git_commit_sha = get_git_commit_sha_short() + + # Check if the git tree is dirty + git_tree_dirty = is_git_tree_dirty() + + # Construct the version string + version = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}+{git_commit_sha}" + + # Append "+dirty" if the git tree is not clean + if git_tree_dirty: + version += ".dirty" + + return version + + +# Example usage +if __name__ == "__main__": + print(get_version()) diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/__init__.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/__init__.py new file mode 100644 index 00000000..341a77c5 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/__init__.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/__init__.py new file mode 100644 index 00000000..341a77c5 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/alpamayo_dataloader.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/alpamayo_dataloader.py new file mode 100644 index 00000000..6c89f64e --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/alpamayo_dataloader.py @@ -0,0 +1,327 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import gc +import logging +import os + +import nemo.collections.physicalai.datasets.dataverse.dataverse.utils.alpamayo.rig_decoder as rig_decoder +import nemo.collections.physicalai.datasets.dataverse.dataverse.utils.alpamayo.transformation as transformation +import numpy as np +import torch +import torch.distributed as dist +import torchvision.transforms as transforms +from einops import rearrange +from nemo.collections.physicalai.datasets.dataverse.dataverse.datasets.base import DataField +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.config_dataverse import DATAVERSE_CONFIG +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.dataloader_utils import ( + dict_collation_fn, +) +from nemo.collections.physicalai.datasets.dataverse_dataset.instantiate_utils import instantiate_from_config +from torch.utils.data import DataLoader + + +try: + from megatron.core import parallel_state + + USE_MEGATRON = True +except ImportError: + USE_MEGATRON = False + +import torch.profiler + + +class DrivingVideoDataLoader(DataLoader): + def __init__(self, dataset, batch_size: int = 1, *args, **kw): + dataset_obj = dataset.build_dataset() + if "dataloaders" in kw: + kw.pop("dataloaders") + super().__init__(dataset_obj, batch_size, collate_fn=dict_collation_fn, *args, **kw) + + +def get_driving_dataset( + dataset_name="alpamayo_v2", +): + return DrivingDataset(dataset_name) + + +class DrivingDataset: + def __init__(self, dataset_name): + self.dataset_name = dataset_name + self.dataset_config = DATAVERSE_CONFIG[dataset_name] + + def build_dataset(self): + # we only create the dataset when we call this function + return InfiniteDataVerse(**self.dataset_config) + + +class InfiniteDataVerse: + def __init__( + self, + dataset_cfg, + batch_size=1, + sample_n_frames=8, + sample_size=[320, 512], + crop_size=None, + load_trajectory=False, + load_frame_repeat=False, + fps=14, + load_video=False, + ): + self.dataset = instantiate_from_config(dataset_cfg) + self.n_data = self.dataset.num_videos() + + self.load_trajectory = load_trajectory + self.load_frame_repeat = load_frame_repeat + self.fps = fps + self.load_video = load_video + # Split the data by node, make sure each node has different data sample + # Ranks of the same pp/tp/cp group will have the same dp rank and thus share the same group id. + if parallel_state.is_initialized(): + dp_group_id = parallel_state.get_data_parallel_rank() + dp_world_size = parallel_state.get_data_parallel_world_size() + logging.critical( + f"Using parallelism size CP :{parallel_state.get_context_parallel_world_size()}, " + + f"TP :{parallel_state.get_tensor_model_parallel_world_size()} for video dataset, " + + f"DP: {dp_group_id}, DP World size: {dp_world_size}" + ) + else: + dp_world_size = 1 + dp_group_id = 0 + self.n_data_per_node = self.n_data // dp_world_size + self.data_start_idx = dp_group_id * self.n_data_per_node + self.dp_group_id = dp_group_id + + # Make an infinite loop + maximum_iter = 1e8 * batch_size # a hack to create infinite loop + self.multiplier = int(maximum_iter // self.n_data_per_node) + + self.sample_n_frames = sample_n_frames + self.sample_size = sample_size + + if crop_size is None: + if self.sample_size != []: + self.crop_size = self.sample_size + else: + self.crop_size = [512, 1024] + else: + self.crop_size = crop_size + + if self.sample_size != []: + self.img_transform = transforms.Compose( + [ + transforms.Resize( + sample_size, interpolation=transforms.InterpolationMode.BILINEAR, antialias=True + ), + transforms.CenterCrop(self.crop_size), + ] + ) + self.norm_image = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True) + + cache_dir = os.environ.get("XDG_CACHE_HOME") + self.camera_t5_embedding = {} + for camera in self.dataset.camkeys: + self.camera_t5_embedding[camera] = torch.load( + os.path.join(cache_dir, "multicamera", f"video_camera_embeddings_v0_{camera}.pt") + ) + + self.camera_text_caption = { + "camera_front_wide_120fov": "The video is captured from a camera mounted on a car. The camera is facing forward.", + "camera_rear_tele_30fov": "The video is captured from a camera mounted on a car. The camera is facing backwards.", + "camera_cross_left_120fov": "The video is captured from a camera mounted on a car. The camera is facing to the left.", + "camera_cross_right_120fov": "The video is captured from a camera mounted on a car. The camera is facing to the right.", + "camera_rear_right_70fov": "The video is captured from a camera mounted on a car. The camera is facing the rear right side.", + "camera_rear_left_70fov": "The video is captured from a camera mounted on a car. The camera is facing the rear left side.", + } + + def __len__(self): + return self.multiplier * self.n_data_per_node + + def transform_data(self, sampled_images): + n_frames, _, H, W = sampled_images.shape # (N, C, H, W) + if self.sample_size != []: + sampled_images = self.img_transform(sampled_images) + sampled_images = self.norm_image(sampled_images) # (N, C, H, W) + sample = { + "video": sampled_images.permute(1, 0, 2, 3).contiguous(), # (C, N, H, W) format for cosmos + "is_preprocessed": True, + } + return sample + + def convert_coordinate(self, xyz, rig_info): + cam_name = "camera_front_wide_120fov" + xyz = rearrange(xyz, "(t c) -> t c", c=3) + rig_info = rig_decoder.decode_rig_info(rig_info) + camera_extrinsics = torch.from_numpy(transformation.sensor_to_rig(rig_info[cam_name])) + + rig_to_camera = np.zeros_like(camera_extrinsics) + rig_to_camera[:3, :3] = camera_extrinsics[:3, :3].T + rig_to_camera[:3, 3] = -camera_extrinsics[:3, :3].T @ camera_extrinsics[:3, 3] + camera_xyz = (xyz @ rig_to_camera[:3, :3].T) + rig_to_camera[:3, 3] + return camera_xyz + + def sample_frame_indices(self, chunk_frame_start, num_samples_in_chunk): + video_fps = 30 # video fps is fixed for alpamayo data + # stride used for subsampling video + stride = int(video_fps / self.fps) + # This is the actual target fps we obtain after subsampling + # Start index is randomly selected in the chunk + if num_samples_in_chunk != int(self.sample_n_frames * stride): + frame_start = chunk_frame_start + int( + np.random.choice(num_samples_in_chunk - int(self.sample_n_frames * stride), 1) + ) + else: + frame_start = chunk_frame_start + + frame_end = frame_start + self.sample_n_frames * stride + + return np.arange(frame_start, frame_end, stride).tolist(), frame_start + + def __getitem__(self, idx): + rank = dist.get_rank() if dist.is_initialized() else 0 + data_idx = (idx % self.n_data_per_node) + self.data_start_idx + assert data_idx < self.n_data + data_fields = [DataField.IMAGE_RGB] + if self.load_trajectory: + data_fields.append(DataField.TRAJECTORY) + + view_indices = [i for i in range(len(self.dataset.camkeys))] + try: + t5_and_meta_data = self.dataset._read_t5_and_meta(video_idx=data_idx, view_idxs=view_indices) + except Exception: + print( + f"RANK {rank}: T5 Meta Data Loading ERROR for video_idx {data_idx}. Skip and continue load the next Video." + ) + return self.__getitem__((idx + 1) % len(self)) + clip_id = self.dataset.clip_names[data_idx] + + available_views = t5_and_meta_data.keys() + if len(available_views) == 0: + print( + f"RANK {rank}: T5 Meta Data Loading ERROR for video_idx {data_idx}. The available_views is empty. Skip and continue load the next Video." + ) + return self.__getitem__((idx + 1) % len(self)) + determin_view_id = min(list(available_views)) + random_selection_chunk_id = 0 + + start_ids = int(t5_and_meta_data[determin_view_id]["meta_data"][2][random_selection_chunk_id]) + end_ids = int(t5_and_meta_data[determin_view_id]["meta_data"][3][random_selection_chunk_id]) + + sample_frame_indices, start_index = self.sample_frame_indices(start_ids, end_ids - start_ids) + + read_data_view_indices = [] + read_data_frame_indices = [] + for view in view_indices: + read_data_frame_indices.extend(sample_frame_indices) + read_data_view_indices.extend([view] * len(sample_frame_indices)) + + if self.load_video: + try: + data = self.dataset._read_raw_video( + video_idx=data_idx, + data_fields=data_fields, + frame_idxs=read_data_frame_indices, + view_idxs=read_data_view_indices, + ) + except Exception: + print( + f"RANK {rank}: Data reading ERROR for video_idx {data_idx}. Skip and continue load the next Video." + ) + return self.__getitem__((idx + 1) % len(self)) + sample = self.transform_data(data[DataField.IMAGE_RGB].clone()) + else: + sample = {} + clip_name = "%d-%03d" % (data_idx, start_index) + caption = "" + try: + dummy_text_embedding = torch.zeros(512 * len(view_indices), 1024) + dummy_text_mask = torch.zeros(512 * len(view_indices)) + raw_embeddings = [] + raw_caption = [] + for view_id in view_indices: + if view_id in t5_and_meta_data: + curr_camera_text = self.camera_text_caption[self.dataset.camkeys[view_id]] + + curr_caption = t5_and_meta_data[view_id]["meta_data"][1][random_selection_chunk_id] + curr_caption = curr_camera_text + curr_caption + text_embedding_np = t5_and_meta_data[view_id]["T5"][random_selection_chunk_id] + raw_embeddings.append(text_embedding_np) + raw_caption.append(t5_and_meta_data[view_id]["meta_data"][1][random_selection_chunk_id]) + text_embedding = torch.from_numpy(text_embedding_np) + curr_camera_t5_embedding = self.camera_t5_embedding[self.dataset.camkeys[view_id]] + + # Concatenate the embeddings + combined_text_embedding = torch.cat([curr_camera_t5_embedding, text_embedding], dim=0) + n_text = combined_text_embedding.shape[0] + + # Ensure the combined embedding does not exceed the maximum size + if n_text > 512: + n_text = 512 + combined_text_embedding = combined_text_embedding[:n_text] + + start_idx = view_id * 512 + end_idx = start_idx + n_text + + # Check for dimension overflow + if end_idx > dummy_text_embedding.shape[0]: + n_text = dummy_text_embedding.shape[0] - start_idx + combined_text_embedding = combined_text_embedding[:n_text] + end_idx = start_idx + n_text + + dummy_text_embedding[start_idx:end_idx] = combined_text_embedding + dummy_text_mask[start_idx:end_idx] = 1 + del text_embedding_np + + else: + curr_camera_text = self.camera_text_caption[self.dataset.camkeys[view_id]] + curr_camera_t5_embedding = self.camera_t5_embedding[self.dataset.camkeys[view_id]] + camera_n_text = curr_camera_t5_embedding.shape[0] + + curr_caption = curr_camera_text + dummy_text_embedding[view_id * 512 : view_id * 512 + camera_n_text] = curr_camera_t5_embedding + dummy_text_mask[view_id * 512 : view_id * 512 + camera_n_text] = 1 + + caption += curr_caption + caption += " ;" + sample["t5_text_embeddings"] = dummy_text_embedding + sample["t5_text_mask"] = dummy_text_mask + sample["t5_raw_text_embeddings"] = raw_embeddings + sample["raw_caption"] = raw_caption + + except Exception: + print( + f"RANK {rank}: Dataloading ERROR for video_idx {data_idx} on T5 loading. Skip and continue load the next Video." + ) + return self.__getitem__((idx + 1) % len(self)) + sample["num_frames"] = self.sample_n_frames + sample["image_size"] = torch.from_numpy(np.asarray(self.crop_size)) + sample["fps"] = self.fps + sample["__key__"] = clip_name + sample["clip_name"] = clip_name + sample["padding_mask"] = torch.zeros(1, self.crop_size[0], self.crop_size[1]) + sample["caption"] = caption + sample["clip_id"] = clip_id + if self.load_frame_repeat: + sample["frame_repeat"] = data["num_repeated_frames"] + + if self.load_trajectory: + # with lvg, use rig coordinate + trajectory = rearrange(data[DataField.TRAJECTORY], "(t c) -> t c", c=3) + trajectory = trajectory / torch.FloatTensor([[10.0, 4.0, 1.0]]) + sample["trajectory"] = rearrange(trajectory, "t c -> (t c)") + + gc.collect() + return sample diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/config_dataverse.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/config_dataverse.py new file mode 100644 index 00000000..448699ac --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/config_dataverse.py @@ -0,0 +1,81 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from omegaconf import DictConfig +from platformdirs import user_cache_path + + +dataset_path = str(user_cache_path("AV-V2.2")) + +DATAVERSE_CONFIG = dict() +tar_dirs_training = [ + f"{dataset_path}/trainv2-2-chunk-00", + f"{dataset_path}/trainv2-2-chunk-01", + f"{dataset_path}/trainv2-2-chunk-02", + f"{dataset_path}/trainv2-2-chunk-03", + f"{dataset_path}/trainv2-2-chunk-04", + f"{dataset_path}/trainv2-2-chunk-05", + f"{dataset_path}/trainv2-2-chunk-06", + f"{dataset_path}/trainv2-2-chunk-07", + f"{dataset_path}/trainv2-2-chunk-08", + f"{dataset_path}/trainv2-2-chunk-09", + f"{dataset_path}/trainv2-2-chunk-10", + f"{dataset_path}/trainv2-2-chunk-11", + f"{dataset_path}/trainv2-2-chunk-12", + f"{dataset_path}/trainv2-2-chunk-13", + f"{dataset_path}/trainv2-2-chunk-14", + f"{dataset_path}/trainv2-2-chunk-15", +] + +tar_dirs_training_full = tar_dirs_training + [ + f"{dataset_path}/trainv2-2-chunk-16", + f"{dataset_path}/trainv2-2-chunk-17", + f"{dataset_path}/trainv2-2-chunk-18", + f"{dataset_path}/trainv2-2-chunk-19", + f"{dataset_path}/trainv2-2-chunk-20", + f"{dataset_path}/trainv2-2-chunk-21", + f"{dataset_path}/trainv2-2-chunk-22", +] +cmeare_groups = [ + "camera_front_wide_120fov", + "camera_cross_left_120fov", + "camera_cross_right_120fov", + "camera_rear_tele_30fov", +] +cmeare_6_group = cmeare_groups + ["camera_rear_left_70fov", "camera_rear_right_70fov"] + + +DATAVERSE_CONFIG["alpamayo_v2_traj_qwen_24fps_6_cameras_frame_repeat"] = { + "dataset_cfg": DictConfig( + { + "target": "nemo.collections.physicalai.datasets.dataverse.dataverse.datasets.cosmos_av_w_traj.CosmosAV", + "params": { + "tar_dirs": tar_dirs_training, + "uuid_dirs": f"{dataset_path}/uuid", + "t5_dirs": f"{dataset_path}/alpamayo_caption_t5/qwen_t5_tars/", + "probe_tar": True, + "camkeys": cmeare_6_group, + "rectify": {"enabled": False}, + "use_hq_data": True, + "decode_traj": True, + }, + } + ), + "sample_n_frames": 57, + "sample_size": [480, 848], + "fps": 24, + "load_video": True, + "load_frame_repeat": True, + "load_trajectory": True, +} diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/dataloader_utils.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/dataloader_utils.py new file mode 100644 index 00000000..2ed08c5f --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/driving_dataloader/dataloader_utils.py @@ -0,0 +1,61 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch + + +def dict_collation_fn(samples, combine_tensors=True, combine_scalars=True, ignore_keys=None): + """Take a list of samples (as dictionary) and create a batch, preserving the keys. + If `tensors` is True, `ndarray` objects are combined into + tensor batches. + :param dict samples: list of samples + :param bool tensors: whether to turn lists of ndarrays into a single ndarray + :returns: single sample consisting of a batch + :rtype: dict + """ + + batched = {key: [] for key in samples[0]} + # assert isinstance(samples[0][first_key], (list, tuple)), type(samples[first_key]) + + for s in samples: + [batched[key].append(s[key]) for key in batched if not (ignore_keys is not None and key in ignore_keys)] + + result = {} + for key in batched: + if ignore_keys and key in ignore_keys: + continue + try: + if isinstance(batched[key][0], bool): + assert key == "is_preprocessed" + result[key] = batched[key][0] # this is a hack to align with cosmos data + elif isinstance(batched[key][0], (int, float)): + if combine_scalars: + result[key] = torch.from_numpy(np.array(list(batched[key]))) + elif isinstance(batched[key][0], torch.Tensor): + if combine_tensors: + result[key] = torch.stack(list(batched[key])) + elif isinstance(batched[key][0], np.ndarray): + if combine_tensors: + result[key] = np.array(list(batched[key])) + elif isinstance(batched[key][0], list) and isinstance(batched[key][0][0], int): + result[key] = [torch.Tensor(elems).long() for elems in zip(*batched[key])] + else: + result[key] = list(batched[key]) + except Exception as e: + print(key) + raise e + # result.append(b) + del batched + return result diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/instantiate_utils.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/instantiate_utils.py new file mode 100644 index 00000000..bc04deae --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/instantiate_utils.py @@ -0,0 +1,38 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import importlib + + +def instantiate_from_config(config, **additional_kwargs): + if "target" not in config: + if config == "__is_first_stage__": + return None + elif config == "__is_unconditional__": + return None + raise KeyError("Expected key `target` to instantiate.") + + # NOTE Kevin changed this to params from kwargs to align with av-vfm + additional_kwargs.update(config.get("params", dict())) + return get_obj_from_str(config["target"])(**additional_kwargs) + + +def get_obj_from_str(string, reload=False): + module, cls = string.rsplit(".", 1) + if reload: + module_imp = importlib.import_module(module) + importlib.reload(module_imp) + return getattr(importlib.import_module(module, package=None), cls) diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/schema.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/schema.py new file mode 100644 index 00000000..a8004aa6 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/schema.py @@ -0,0 +1,45 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import attrs +from imaginaire import config +from imaginaire.config import make_freezable + + +@make_freezable +@attrs.define(slots=False) +class DatasetInfo: + object_store_config: config.ObjectStoreConfig # Object strore config + wdinfo: list[str] # List of wdinfo files + opts: dict = {} # Additional dataset info args + per_dataset_keys: list[str] = [] # List of keys per dataset + source: str = "" # data source + + +@make_freezable +@attrs.define(slots=False) +class DatasetConfig: + keys: list[str] # List of keys used + buffer_size: int # Buffer size used by each worker + dataset_info: list[DatasetInfo] # List of dataset info files, one for each dataset + decoders: list # List of decoder functions for decoding bytestream + streaming_download: bool = True # Whether to use streaming loader + remove_extension_from_keys: bool = True # True: objects will have a key of data_type; False: data_type.extension + sample_keys_full_list_path: Optional[str] = ( + None # Path to the file containing all keys present in the dataset, e.g., "index" + ) diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/utils.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/utils.py new file mode 100644 index 00000000..0b1ac560 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/utils.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import mediapy as media +import numpy as np +import torch +import torchvision +from einops import rearrange + + +def save_videos_grid(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=8): + videos = rearrange(videos, "b c t h w -> t b c h w") + outputs = [] + for x in videos: + x = torchvision.utils.make_grid(x, nrow=n_rows) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + if rescale: + x = (x + 1.0) / 2.0 # -1,1 -> 0,1 + x = torch.clamp(x, 0, 1) + x = (x * 255).numpy().astype(np.uint8) + outputs.append(x) + os.makedirs(os.path.dirname(path), exist_ok=True) + outputs = np.stack(outputs) + media.write_video(path, outputs, fps=fps) diff --git a/nemo_vfm/physicalai/datasets/dataverse_dataset/video_size.py b/nemo_vfm/physicalai/datasets/dataverse_dataset/video_size.py new file mode 100644 index 00000000..099d0aac --- /dev/null +++ b/nemo_vfm/physicalai/datasets/dataverse_dataset/video_size.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VIDEO_RES_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": { # 1080p doesn't have 1:1 + "1,1": (1024, 1024), + "4,3": (1440, 1056), + "3,4": (1056, 1440), + "16,9": (1920, 1056), + "9,16": (1056, 1920), + }, + # 1024; the video format does not support it, but here we match it with image resolution + "1024": {"1,1": (1024, 1024), "4,3": (1280, 1024), "3,4": (1024, 1280), "16,9": (1280, 768), "9,16": (768, 1280)}, + "720": {"1,1": (960, 960), "4,3": (960, 704), "3,4": (704, 960), "16,9": (1280, 704), "9,16": (704, 1280)}, + "512": {"1,1": (512, 512), "4,3": (640, 512), "3,4": (512, 640), "16,9": (640, 384), "9,16": (384, 640)}, + "256": { + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (320, 192), + "9,16": (192, 320), + }, +} diff --git a/nemo_vfm/physicalai/datasets/mock_dataset.py b/nemo_vfm/physicalai/datasets/mock_dataset.py new file mode 100644 index 00000000..2ce89c84 --- /dev/null +++ b/nemo_vfm/physicalai/datasets/mock_dataset.py @@ -0,0 +1,184 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import inspect +from typing import Any, Callable, Dict + +import torch +from torch.utils.data import Dataset + + +MAX_LENGTH = 1 << 15 + + +class LambdaDataset(torch.utils.data.Dataset): + """ + A dataset that generates items by applying a function. This allows for creating + dynamic datasets where the items are the result of function calls. The function can optionally + accept an index argument. + + Attributes: + length (int): The total number of items in the dataset. + fn (Callable): The function to generate dataset items. + is_index_in_params (bool): Flag to determine whether 'index' should be passed + to the function `fn`. + """ + + def __init__(self, fn: Callable, length: int = MAX_LENGTH) -> None: + """ + Initializes the LambdaDataset with a function and the total length. + + Args: + fn (Callable): A function that returns a dataset item. It can optionally accept an + index argument to generate data items based on their index. + length (int): The total number of items in the dataset, defaults to MAX_LENGTH. + """ + self.length = length + self.fn = fn + + try: + # Attempt to inspect the function signature to determine if it accepts an 'index' parameter. + signature = inspect.signature(fn) + self.is_index_in_params = "index" in signature.parameters + except ValueError: + # If the function signature is not inspectable, assume 'index' is not a parameter. + self.is_index_in_params = False + + def __len__(self) -> int: + """ + Returns the total length of the dataset. + + Returns: + int: The number of items in the dataset. + """ + return self.length + + def __getitem__(self, index: int) -> Any: + """ + Retrieves an item at a specific index from the dataset by calling the function `fn`. + Passes the index to `fn` if `fn` is designed to accept an index. + + Args: + index (int): The index of the item to retrieve. + + Returns: + Any: The item returned by the function `fn`. + """ + if self.is_index_in_params: + return self.fn(index) # Call fn with index if it accepts an index parameter. + return self.fn() # Call fn without any parameters if it does not accept the index. + + +class RepeatDataset(torch.utils.data.Dataset): + """ + A dataset wrapper that allows repeating access to items from an underlying dataset. + + This dataset can be used to create an artificial extension of the underlying dataset + to a specified `length`. Each item from the original dataset can be accessed + repeatedly up to `num_item` times before it loops back. + + Attributes: + length (int): The total length of the dataset to be exposed. + dataset (Dataset): The original dataset. + num_item (int): Number of times each item is repeated. + cache_item (dict): Cache to store accessed items to avoid recomputation. + """ + + def __init__(self, dataset: Dataset, length: int = MAX_LENGTH, num_item: int = 1) -> None: + """ + Initializes the RepeatDataset with a dataset, length, and number of repeats per item. + + Args: + dataset (Dataset): The dataset to repeat. + length (int): The total length of the dataset to be exposed. Defaults to MAX_LENGTH. + num_item (int): The number of times to repeat each item. Defaults to 1. + """ + self.length = length + self.dataset = dataset + self.num_item = num_item + self.cache_item = {} + + def __len__(self) -> int: + return self.length + + def __getitem__(self, index: int) -> Any: + index = index % self.num_item + if index not in self.cache_item: + self.cache_item[index] = self.dataset[index] + return self.cache_item[index] + + +class CombinedDictDataset(torch.utils.data.Dataset): + """ + A dataset that wraps multiple PyTorch datasets and returns a dictionary of data items from each dataset for a given index. + This dataset ensures that all constituent datasets have the same length by setting the length to the minimum length of the datasets provided. + + Parameters: + ----------- + **datasets : Dict[str, Dataset] + A dictionary where keys are string identifiers for the datasets and values are the datasets instances themselves. + + Attributes: + ----------- + datasets : Dict[str, Dataset] + Stores the input datasets. + max_length : int + The minimum length among all provided datasets, determining the length of this combined dataset. + + Examples: + --------- + >>> dataset1 = torch.utils.data.TensorDataset(torch.randn(100, 3, 32, 32)) + >>> dataset2 = torch.utils.data.TensorDataset(torch.randn(100, 3, 32, 32)) + >>> combined_dataset = CombinedDictDataset(dataset1=dataset1, dataset2=dataset2) + >>> print(len(combined_dataset)) + 100 + >>> data = combined_dataset[50] + >>> print(data.keys()) + dict_keys(['dataset1', 'dataset2']) + """ + + def __init__(self, **datasets: Dict[str, Dataset]) -> None: + """ + Initializes the CombinedDictDataset with multiple datasets. + + Args: + **datasets (Dict[str, Dataset]): Key-value pairs where keys are dataset names and values + are dataset instances. Each key-value pair adds a dataset + under the specified key. + """ + self.datasets = datasets + self.max_length = min([len(dataset) for dataset in datasets.values()]) + + def __len__(self) -> int: + return self.max_length + + def __getitem__(self, index: int) -> Dict[str, Any]: + """ + Retrieves an item from each dataset at the specified index, combines them into a dictionary, + and returns the dictionary. Each key in the dictionary corresponds to one of the dataset names provided + during initialization, and its value is the item from that dataset at the given index. + + Args: + index (int): The index of the items to retrieve across all datasets. + + Returns: + Dict[str, Any]: A dictionary containing data items from all datasets for the given index. + Each key corresponds to a dataset name, and its value is the data item from that dataset. + """ + data = {} + for key, dataset in self.datasets.items(): + data[key] = dataset[index] + return data diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/README.md b/nemo_vfm/physicalai/diffusion/post_training/multicamera/README.md new file mode 100644 index 00000000..d6b4d408 --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/README.md @@ -0,0 +1,93 @@ +# Cosmos Multicamera Diffusion Post-Training: User Guide + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the minimum compute required to run the model(s), as listed in the model support matrix. + - **Containerization Platform**: We recommend using Docker with NVIDIA Container Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Allocate Resources and Run the Container + +Run the following command to allocate a node and launch the container. you can build container using `nemo/collections/physicalai/Dockerfile` +```bash +docker build -f nemo/collections/physicalai/Dockerfile . +``` + + +## Multi-Node + +### 3. Modify the Environment Variables Inside of the Training Script + +```bash +# nemo/collections/physicalai/diffusion/post_training/multicamera +export HF_TOKEN="your huggingface access token" +export WANDB_API_KEY="your wandb API key" +export WANDB_PROJECT="name of your wandb project" + +# use these default paths unless you have your own model/data for post-training +export HF_HOME=/path/to/.cache/huggingface +export XDG_CACHE_HOME=/path/to/.cache +``` + +### 4. Launch the Training Script using SBATCH + +```bash +# multinode scheduled +sbatch -A coreai_dlalgo_llm -N16 --tasks-per-node=8 --gpus-per-node=8 -p polar,polar3,polar4 --dependency=singleton --signal=TERM@600 -t 4:00:0 nemo/collections/physicalai/diffusion/post_training/multicamera/train.sh +``` + +### OR + +### 4. Manually Allocate the Node and Launch the Training Script + +```bash +# Can replace the 1 in -N1 with the number of desired nodes +salloc -A coreai_dlalgo_llm -N1 --tasks-per-node=8 --gpus-per-node=8 -p interactive -t 4:00:0 +bash nemo/collections/physicalai/diffusion/post_training/multicamera/train.sh +``` + + +# Single Node + +### 3. Allocate Resources and Run the Container + +Run the following command to allocate a node and launch the container + +```bash +salloc -A coreai_dlalgo_llm -N1 --tasks-per-node=8 --gpus-per-node=8 -p interactive -t 4:00:0 +srun -N1 --tasks-per-node 1 --container-image path/to/container --container-mounts "/lustre:/lustre/,/home:/home" --no-container-mount-home --pty bash +``` + +### 4. Configure Environment Variables Inside of the Container + +navigate into NeMo repo + +```bash +# Install packages +pip install lru-dict pyquaternion git+https://github.com/NVIDIA/NeMo-Run.git@f07877f80dbfc91c4d37683864cf2552b96d62f1 +pip install -U moviepy==1.0.3 + +# Export environment variables +export PYTHONPATH=$PYTHONPATH:nemo/collections/physicalai/datasets/dataverse +export HF_TOKEN="your huggingface access token" +export WANDB_API_KEY="your wandb API key" +export WANDB_PROJECT="name of your wandb project" +export WANDB_RESUME=allow +export NVTE_FUSED_ATTN=0 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +# use these default paths unless you have your own model/data for post-training +export HF_HOME=/path/to/.cache/huggingface +export XDG_CACHE_HOME=/path/to/.cache +``` + +### 5. Post-Train the model +```bash +torchrun --nproc_per_node=8 nemo/collections/physicalai/diffusion/post_training/multicamera/multicamera.py +``` + diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/dit_multi_camera.py b/nemo_vfm/physicalai/diffusion/post_training/multicamera/dit_multi_camera.py new file mode 100644 index 00000000..86482b6a --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/dit_multi_camera.py @@ -0,0 +1,1096 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import logging +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional, Tuple + +import attrs +import einops +import numpy as np +import torch +import wandb +from einops import rearrange, repeat +from megatron.core import InferenceParams, parallel_state, tensor_parallel +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.packed_seq_params import PackedSeqParams +from nemo.collections.diffusion.models.dit.dit_model_7b import ( + DiTCrossAttentionModel7B, + PatchEmbed, + cat_outputs_cp, + get_1d_sincos_pos_embed_from_grid, + split_inputs_cp, +) +from nemo.collections.diffusion.models.model import DiT7BConfig, DiTModel, dynamic_import +from nemo.collections.diffusion.sampler.conditioner import ( + AbstractEmbModel, + DataType, + Edify4Condition, + TrainingOnlyEmbModel, + VideoConditioner, +) +from nemo.collections.diffusion.sampler.conditioner_configs import ( + FPSConfig, + ImageSizeConfig, + NumFramesConfig, + PaddingMaskConfig, + TextConfig, +) +from nemo.collections.diffusion.sampler.cosmos.cosmos_diffusion_pipeline import CosmosDiffusionPipeline +from nemo.collections.diffusion.sampler.res.res_sampler import COMMON_SOLVER_OPTIONS +from torch import Tensor, nn +from torch.distributed import ProcessGroup, get_process_group_ranks +from torchvision import transforms +from typing_extensions import override + + +class MultiCameraVideoPositionEmb(nn.Module): + def __init__( + self, + ): + super().__init__() + self.cp_group = None + + def enable_context_parallel(self, cp_group: ProcessGroup): + self.cp_group = cp_group + + def disable_context_parallel(self): + self.cp_group = None + + def forward(self, x_B_T_H_W_C: torch.Tensor, fps=Optional[torch.Tensor]) -> torch.Tensor: + """ + With CP, the function assume that the input tensor is already split. It delegates the embedding generation to generate_embeddings function. + """ + B_T_H_W_C = x_B_T_H_W_C.shape + if self.cp_group is not None: + cp_ranks = get_process_group_ranks(self.cp_group) + cp_size = len(cp_ranks) + B, T, H, W, C = B_T_H_W_C + B_T_H_W_C = (B, T * cp_size, H, W, C) + embeddings = self.generate_embeddings(B_T_H_W_C, fps=fps) + + if self.cp_group is not None: + if isinstance(self, MultiCameraVideoRopePosition3DEmb): + seq_dim = 1 + embeddings = rearrange(embeddings, "(V T) H W D -> V (T H W) 1 1 D", V=self.n_cameras).float() + # rearrange(em_T_H_W_D, "t h w d -> (t h w) 1 1 d").float() + embeddings = split_inputs_cp(x=embeddings, seq_dim=seq_dim, cp_group=self.cp_group) + embeddings = rearrange(embeddings, "V T 1 1 D -> (V T) 1 1 D", V=self.n_cameras).float() + else: + seq_dim = 1 + embeddings = rearrange(embeddings, "B (V T) H W C -> (B V) T H W C", V=self.n_cameras) + embeddings = split_inputs_cp(x=embeddings, seq_dim=seq_dim, cp_group=self.cp_group) + embeddings = rearrange(embeddings, "(B V) T H W C -> B (V T) H W C", V=self.n_cameras) + else: + if isinstance(self, MultiCameraVideoRopePosition3DEmb): + embeddings = rearrange(embeddings, "t h w d -> (t h w) 1 1 d").float() + + return embeddings + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]): + raise NotImplementedError + + +class MultiCameraVideoRopePosition3DEmb(MultiCameraVideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + head_dim: int, + len_h: int, + len_w: int, + len_t: int, + base_fps: int = 24, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + n_cameras: int = 4, + **kwargs, # used for compatibility with other positional embeddings; unused in this class + ): + del kwargs + super().__init__() + self.register_buffer("seq", torch.arange(max(len_h, len_w, len_t), dtype=torch.float)) + self.base_fps = base_fps + self.max_h = len_h + self.max_w = len_w + self.n_cameras = n_cameras + dim = head_dim + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + self.register_buffer( + "dim_spatial_range", + torch.arange(0, dim_h, 2)[: (dim_h // 2)].float().cuda() / dim_h, + persistent=False, + ) + self.register_buffer( + "dim_temporal_range", + torch.arange(0, dim_t, 2)[: (dim_t // 2)].float().cuda() / dim_t, + persistent=False, + ) + + self.h_ntk_factor = h_extrapolation_ratio ** (dim_h / (dim_h - 2)) + self.w_ntk_factor = w_extrapolation_ratio ** (dim_w / (dim_w - 2)) + self.t_ntk_factor = t_extrapolation_ratio ** (dim_t / (dim_t - 2)) + + def generate_embedding_for_batch( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. Defaults to None. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. Defaults to None. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. Defaults to None. + + Returns: + Not specified in the original code snippet. + """ + h_ntk_factor = h_ntk_factor if h_ntk_factor is not None else self.h_ntk_factor + w_ntk_factor = w_ntk_factor if w_ntk_factor is not None else self.w_ntk_factor + t_ntk_factor = t_ntk_factor if t_ntk_factor is not None else self.t_ntk_factor + + h_theta = 10000.0 * h_ntk_factor + w_theta = 10000.0 * w_ntk_factor + t_theta = 10000.0 * t_ntk_factor + + h_spatial_freqs = 1.0 / (h_theta**self.dim_spatial_range) + w_spatial_freqs = 1.0 / (w_theta**self.dim_spatial_range) + temporal_freqs = 1.0 / (t_theta**self.dim_temporal_range) + + B, T, H, W, _ = B_T_H_W_C + uniform_fps = (fps is None) or (fps.min() == fps.max()) + assert uniform_fps # only support uniform fps now + + assert uniform_fps or B == 1 or T == 1, ( + "For video batch, batch size should be 1 for non-uniform fps. For image batch, T should be 1" + ) + assert H <= self.max_h and W <= self.max_w, ( + f"Input dimensions (H={H}, W={W}) exceed the maximum dimensions (max_h={self.max_h}, max_w={self.max_w}) configured for positional embedding. Please adjust the input size or increase the maximum dimensions in the model configuration." + ) + half_emb_h = torch.outer(self.seq[:H], h_spatial_freqs) + half_emb_w = torch.outer(self.seq[:W], w_spatial_freqs) + + # apply sequence scaling in temporal dimension + if fps is None: # image case + assert T == 1, "T should be 1 for image batch." + half_emb_t = torch.outer(self.seq[:T], temporal_freqs) + else: + half_emb_t = torch.outer(self.seq[:T] / fps[:1] * self.base_fps, temporal_freqs) + + em_T_H_W_D = torch.cat( + [ + repeat(half_emb_t, "t d -> t h w d", h=H, w=W), + repeat(half_emb_h, "h d -> t h w d", t=T, w=W), + repeat(half_emb_w, "w d -> t h w d", t=T, h=H), + ] + * 2, + dim=-1, + ) + + return em_T_H_W_D + + def generate_embeddings( + self, + B_T_H_W_C: torch.Size, + fps: Optional[torch.Tensor] = None, + h_ntk_factor: Optional[float] = None, + w_ntk_factor: Optional[float] = None, + t_ntk_factor: Optional[float] = None, + ): + """ + Generate embeddings for the given input size. The camera view dimension is merged in the T dimension + + Args: + B_T_H_W_C (torch.Size): Input tensor size (Batch, Time * Views, Height, Width, Channels). + fps (Optional[torch.Tensor], optional): Frames per second. Defaults to None. + h_ntk_factor (Optional[float], optional): Height NTK factor. If None, uses self.h_ntk_factor. Defaults to None. + w_ntk_factor (Optional[float], optional): Width NTK factor. If None, uses self.w_ntk_factor. Defaults to None. + t_ntk_factor (Optional[float], optional): Time NTK factor. If None, uses self.t_ntk_factor. Defaults to None. + + Returns: + Not specified in the original code snippet. + """ + + B, T, H, W, C = B_T_H_W_C + + single_camera_B_T_H_W_C = (B, T // self.n_cameras, H, W, C) + em_T_H_W_D = torch.cat( + [ + self.generate_embedding_for_batch( + single_camera_B_T_H_W_C, + fps=fps, + h_ntk_factor=h_ntk_factor, + w_ntk_factor=w_ntk_factor, + t_ntk_factor=t_ntk_factor, + ) + for item in range(self.n_cameras) + ], + dim=0, + ) + + return em_T_H_W_D + # return rearrange(em_T_H_W_D, "t h w d -> (t h w) 1 1 d").float() + + +class MultiCameraSinCosPosEmbAxis(MultiCameraVideoPositionEmb): + def __init__( + self, + *, # enforce keyword arguments + interpolation: str, + model_channels: int, + len_h: int, + len_w: int, + len_t: int, + h_extrapolation_ratio: float = 1.0, + w_extrapolation_ratio: float = 1.0, + t_extrapolation_ratio: float = 1.0, + n_cameras: int = 4, + **kwargs, + ): + # TODO: (qsh 2024-11-08) add more interpolation methods and args for extrapolation fine-tuning + """ + Args: + interpolation (str): we curretly only support "crop", ideally when we need extrapolation capacity, we should adjust frequency or other more advanced methods. they are not implemented yet. + """ + del kwargs # unused + self.n_cameras = n_cameras + super().__init__() + self.interpolation = interpolation + assert self.interpolation in ["crop"], f"Unknown interpolation method {self.interpolation}" + + dim = model_channels + dim_h = dim // 6 * 2 + dim_w = dim_h + dim_t = dim - 2 * dim_h + assert dim == dim_h + dim_w + dim_t, f"bad dim: {dim} != {dim_h} + {dim_w} + {dim_t}" + + # rescale pos id is equivalent to rescale frequency + emb_h = get_1d_sincos_pos_embed_from_grid(dim_h, pos=np.arange(len_h) * 1.0 / h_extrapolation_ratio) + emb_w = get_1d_sincos_pos_embed_from_grid(dim_w, pos=np.arange(len_w) * 1.0 / w_extrapolation_ratio) + emb_t = get_1d_sincos_pos_embed_from_grid(dim_t, pos=np.arange(len_t) * 1.0 / t_extrapolation_ratio) + + self.register_buffer("pos_emb_h", torch.from_numpy(emb_h).float(), persistent=False) + self.register_buffer("pos_emb_w", torch.from_numpy(emb_w).float(), persistent=False) + self.register_buffer("pos_emb_t", torch.from_numpy(emb_t).float(), persistent=False) + + def generate_embeddings(self, B_T_H_W_C: torch.Size, fps=Optional[torch.Tensor]) -> torch.Tensor: + B, T, H, W, C = B_T_H_W_C + + single_camera_T = T // self.n_cameras + + if self.interpolation == "crop": + emb_h_H = self.pos_emb_h[:H] + emb_w_W = self.pos_emb_w[:W] + emb_t_T = self.pos_emb_t[:single_camera_T] + emb = torch.cat( + [ + torch.cat( + [ + repeat(emb_t_T, "t d-> b t h w d", b=B, h=H, w=W), + repeat(emb_h_H, "h d-> b t h w d", b=B, t=single_camera_T, w=W), + repeat(emb_w_W, "w d-> b t h w d", b=B, t=single_camera_T, h=H), + ], + dim=-1, + ) + for _ in range(self.n_cameras) + ], + 1, + ) + assert list(emb.shape)[:4] == [B, T, H, W], f"bad shape: {list(emb.shape)[:4]} != {B, T, H, W}" + return emb + + raise ValueError(f"Unknown interpolation method {self.interpolation}") + + +class MultiCameraDiTCrossAttentionModel7B(DiTCrossAttentionModel7B): + """DiT with CrossAttention model. + + Args: + config (TransformerConfig): transformer config + + transformer_decoder_layer_spec (ModuleSpec): transformer layer customization specs for decoder + + pre_process (bool): Include embedding layer (used with pipeline parallelism) + post_process (bool): Include an output layer (used with pipeline parallelism) + + fp16_lm_cross_entropy (bool, optional): Defaults to False + + parallel_output (bool): Do not gather the outputs, keep them split across tensor parallel ranks + + share_embeddings_and_output_weights (bool): When True, input embeddings and output logit weights are + shared. Defaults to False. + + position_embedding_type (string): Position embedding type. Options ['learned_absolute', 'rope']. + Defaults is 'learned_absolute'. + + rotary_percent (float): Percent of rotary dimension to use for rotary position embeddings. + Defaults to 1.0 (100%). Ignored unless position_embedding_type is 'rope'. + + seq_len_interpolation_factor (float): scale of linearly interpolating RoPE for longer sequences. + The value must be a float larger than 1.0. Defaults to None. + """ + + def __init__( + self, + *args, + n_cameras=4, + camera_condition_dim=4, + traj_condition_dim=0, + add_repeat_frame_embedding=True, + concat_camera_embedding: bool = True, + concat_traj_embedding=False, + in_channels=16, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.n_cameras = n_cameras + self.camera_condition_dim = camera_condition_dim + self.concat_camera_embedding = concat_camera_embedding + self.traj_condition_dim = traj_condition_dim + self.concat_traj_embedding = concat_traj_embedding + self.add_repeat_frame_embedding = add_repeat_frame_embedding + + if self.concat_camera_embedding: + in_channels = in_channels + camera_condition_dim if camera_condition_dim > 0 else in_channels + + if self.concat_traj_embedding: + in_channels = in_channels + traj_condition_dim if traj_condition_dim > 0 else in_channels + + in_channels = in_channels + 1 if self.concat_padding_mask else in_channels + + self.x_embedder = ( + PatchEmbed( + spatial_patch_size=self.patch_spatial, + temporal_patch_size=self.patch_temporal, + in_channels=in_channels, + out_channels=self.config.hidden_size, + bias=False, + keep_spatio=True, + legacy_patch_emb=self.legacy_patch_emb, + ) + .cuda() + .to(dtype=torch.bfloat16) + ) + + if self.extra_per_block_abs_pos_emb: + self.extra_pos_embedder = MultiCameraSinCosPosEmbAxis( + h_extrapolation_ratio=1, + w_extrapolation_ratio=1, + t_extrapolation_ratio=1, + model_channels=self.config.hidden_size, + len_h=self.max_img_h // self.patch_spatial, + len_w=self.max_img_w // self.patch_spatial, + len_t=self.max_frames // self.patch_temporal, + interpolation=self.pos_emb_interpolation, + ) + + self.view_embeddings = nn.Embedding(n_cameras, camera_condition_dim) # Learnable embedding layer + if self.concat_traj_embedding: + self.traj_embeddings = nn.Linear(192, self.traj_condition_dim) # Learnable embedding layer + if self.add_repeat_frame_embedding: + self.repeat_frame_embedding = nn.Linear(1, camera_condition_dim) # Learnable embedding layer + + def prepare_embedded_sequence( + self, + x_B_C_T_H_W: torch.Tensor, + fps: Optional[torch.Tensor] = None, + padding_mask: Optional[torch.Tensor] = None, + trajectory: Optional[torch.Tensor] = None, + frame_repeat: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.concat_padding_mask: + padding_mask = padding_mask.squeeze(0) + padding_mask = transforms.functional.resize( + padding_mask, list(x_B_C_T_H_W.shape[-2:]), interpolation=transforms.InterpolationMode.NEAREST + ) + x_B_C_T_H_W = torch.cat( + [x_B_C_T_H_W, padding_mask.unsqueeze(1).repeat(1, 1, x_B_C_T_H_W.shape[2], 1, 1)], dim=1 + ) + + # ********* multicamera ********* + view_indices = torch.arange(self.n_cameras).to(x_B_C_T_H_W.device) # View indices [0, 1, ..., V-1] + view_embedding = self.view_embeddings(view_indices) # Shape: [V, embedding_dim] + view_embedding = rearrange(view_embedding, "V D -> D V") + view_embedding = ( + view_embedding.unsqueeze(0).unsqueeze(3).unsqueeze(4).unsqueeze(5) + ) # Shape: [1, D, V, 1, 1, 1] + + if self.add_repeat_frame_embedding: + if frame_repeat is None: + frame_repeat = ( + torch.zeros([x_B_C_T_H_W.shape[0], view_embedding.shape[1]]) + .to(view_embedding.device) + .to(view_embedding.dtype) + ) + frame_repeat_embedding = self.repeat_frame_embedding(frame_repeat.unsqueeze(-1)) + frame_repeat_embedding = rearrange(frame_repeat_embedding, "B V D -> B D V") + view_embedding = view_embedding + frame_repeat_embedding.unsqueeze(3).unsqueeze(4).unsqueeze(5) + + x_B_C_V_T_H_W = rearrange(x_B_C_T_H_W, "B C (V T) H W -> B C V T H W", V=self.n_cameras) + view_embedding = view_embedding.expand( + x_B_C_V_T_H_W.shape[0], + view_embedding.shape[1], + view_embedding.shape[2], + x_B_C_V_T_H_W.shape[3], + x_B_C_V_T_H_W.shape[4], + x_B_C_V_T_H_W.shape[5], + ) # Shape: [B, V, 3, t, H, W] + if self.concat_traj_embedding: + traj_emb = self.traj_embeddings(trajectory) + traj_emb = traj_emb.unsqueeze(2).unsqueeze(3).unsqueeze(4).unsqueeze(5) + traj_emb = traj_emb.expand( + x_B_C_V_T_H_W.shape[0], + traj_emb.shape[1], + view_embedding.shape[2], + x_B_C_V_T_H_W.shape[3], + x_B_C_V_T_H_W.shape[4], + x_B_C_V_T_H_W.shape[5], + ) # Shape: [B, V, 3, t, H, W] + + x_B_C_V_T_H_W = torch.cat([x_B_C_V_T_H_W, view_embedding, traj_emb], dim=1) + else: + x_B_C_V_T_H_W = torch.cat([x_B_C_V_T_H_W, view_embedding], dim=1) + + x_B_C_T_H_W = rearrange(x_B_C_V_T_H_W, " B C V T H W -> B C (V T) H W", V=self.n_cameras) + # ********* multicamera ********* + + x_B_T_H_W_D = self.x_embedder(x_B_C_T_H_W) + if self.extra_per_block_abs_pos_emb: + extra_pos_emb = self.extra_pos_embedder(x_B_T_H_W_D, fps=fps) + else: + extra_pos_emb = None + + if "rope" in self.pos_emb_cls.lower(): + if extra_pos_emb is not None: + extra_pos_emb = rearrange(extra_pos_emb, "B T H W D -> (T H W) B D") + return x_B_T_H_W_D, [self.pos_embedder(x_B_T_H_W_D, fps=fps), extra_pos_emb] + else: + return x_B_T_H_W_D, self.pos_embedder(x_B_T_H_W_D, fps=fps) + + if "fps_aware" in self.pos_emb_cls: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D, fps=fps.cuda()) # [B, T, H, W, D] + else: + x_B_T_H_W_D = x_B_T_H_W_D + self.pos_embedder(x_B_T_H_W_D) # [B, T, H, W, D] + return x_B_T_H_W_D, None + + def forward( + self, + x: Tensor, + timesteps: Tensor, + crossattn_emb: Tensor, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + pos_ids: Tensor = None, + **kwargs, + ) -> Tensor: + """Forward pass. + + Args: + x (Tensor): vae encoded videos (b s c) + encoder_decoder_attn_mask (Tensor): cross-attention mask between encoder and decoder + inference_params (InferenceParams): relevant arguments for inferencing + + Returns: + Tensor: loss tensor + """ + # Decoder forward + # Decoder embedding. + # print(f'x={x}') + # x = x.squeeze(0) + original_shape = x.shape + B, C, T, H, W = original_shape + + fps = kwargs.get("fps", None) + if len(fps.shape) > 1: + fps = fps.squeeze(0) + padding_mask = kwargs.get("padding_mask", None) + image_size = kwargs.get("image_size", None) + trajectory = kwargs.get("trajectory", None) + frame_repeat = kwargs.get("frame_repeat", None) + + if self.pre_process: + x_B_T_H_W_D, rope_emb_L_1_1_D = self.prepare_embedded_sequence( + x, fps=fps, padding_mask=padding_mask, trajectory=trajectory, frame_repeat=frame_repeat + ) + B, T, H, W, D = x_B_T_H_W_D.shape + # print(f'x_T_H_W_B_D.shape={x_T_H_W_B_D.shape}') + x_S_B_D = rearrange(x_B_T_H_W_D, "B T H W D -> (T H W) B D") + # print(f'x_S_B_D.shape={x_S_B_D.shape}') + else: + # intermediate stage of pipeline + x_S_B_D = None # should it take encoder_hidden_states + + _, _, D = x_S_B_D.shape + + # print(f'x_S_B_D={x_S_B_D}') + + # logging affline scale information + affline_scale_log_info = {} + + timesteps_B_D, adaln_lora_B_3D = self.t_embedder(timesteps.flatten()) + affline_emb_B_D = timesteps_B_D + affline_scale_log_info["timesteps_B_D"] = timesteps_B_D.detach() + + if self.additional_timestamp_channels: + if type(image_size) == tuple: + image_size = image_size[0] + additional_cond_B_D = self.prepare_additional_timestamp_embedder( + bs=x.shape[0], + fps=fps, + h=image_size[:, 0], + w=image_size[:, 1], + org_h=image_size[:, 2], + org_w=image_size[:, 3], + ) + + affline_emb_B_D += additional_cond_B_D + affline_scale_log_info["additional_cond_B_D"] = additional_cond_B_D.detach() + + affline_scale_log_info["affline_emb_B_D"] = affline_emb_B_D.detach() + affline_emb_B_D = self.affline_norm(affline_emb_B_D) + + crossattn_emb = rearrange(crossattn_emb, "B S D -> S B D") + + # [Parth] Enable Sequence Parallelism + if self.config.sequence_parallel: + if self.pre_process: + x_S_B_D = tensor_parallel.scatter_to_sequence_parallel_region(x_S_B_D) + if len(rope_emb_L_1_1_D) > 1: + rope_emb_L_1_1_D[1] = tensor_parallel.scatter_to_sequence_parallel_region(rope_emb_L_1_1_D[1]) + crossattn_emb = tensor_parallel.scatter_to_sequence_parallel_region(crossattn_emb) + # `scatter_to_sequence_parallel_region` returns a view, which prevents + # the original tensor from being garbage collected. Clone to facilitate GC. + # Has a small runtime cost (~0.5%). + if self.config.clone_scatter_output_in_embedding: + if self.pre_process: + x_S_B_D = x_S_B_D.clone() + rope_emb_L_1_1_D[1] = rope_emb_L_1_1_D[1].clone() + crossattn_emb = crossattn_emb.clone() + + packed_seq_params = { + "adaln_lora_B_3D": adaln_lora_B_3D.detach(), + "extra_pos_emb": rope_emb_L_1_1_D[1].detach(), + } + x_S_B_D = self.decoder( + hidden_states=x_S_B_D, + attention_mask=affline_emb_B_D, + context=crossattn_emb, + context_mask=None, + packed_seq_params=packed_seq_params, + rotary_pos_emb=rope_emb_L_1_1_D[0], + ) + # Return if not post_process + if not self.post_process: + return x_S_B_D + + if self.config.sequence_parallel: + x_S_B_D = tensor_parallel.gather_from_sequence_parallel_region(x_S_B_D) + + x_B_T_H_W_D = rearrange(x_S_B_D, "(T H W) B D -> B T H W D", B=B, T=T, H=H, W=W, D=D) + x_B_T_H_W_D = self.decoder_head(x_B_T_H_W_D, affline_emb_B_D, None, original_shape, None, adaln_lora_B_3D) + + return x_B_T_H_W_D + + def sharded_state_dict( + self, prefix: str = "", sharded_offsets: tuple = (), metadata: Optional[Dict] = None + ) -> ShardedStateDict: + """Sharded state dict implementation for GPTModel backward-compatibility (removing extra state). + + Args: + prefix (str): Module name prefix. + sharded_offsets (tuple): PP related offsets, expected to be empty at this module level. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. + + Returns: + ShardedStateDict: sharded state dict for the GPTModel + """ + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) + + sharded_state_dict[f"{prefix}x_embedder.proj.1.weight"].allow_shape_mismatch = True + + return sharded_state_dict + + +class VideoExtendMultiCameraDiTCrossAttentionModel7B(MultiCameraDiTCrossAttentionModel7B): + def __init__(self, *args, in_channels=16, **kwargs): + # extra channel for video condition mask + super().__init__(*args, in_channels=in_channels + 1, **kwargs) + logging.info(f"VideoExtendGeneralDIT in_channels: {in_channels + 1}") + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + crossattn_emb: torch.Tensor, + data_type: Optional[DataType] = DataType.VIDEO, + video_cond_bool: Optional[torch.Tensor] = None, + condition_video_indicator: Optional[torch.Tensor] = None, + condition_video_input_mask: Optional[torch.Tensor] = None, + condition_video_augment_sigma: Optional[torch.Tensor] = None, + condition_video_pose: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """Args: + condition_video_augment_sigma: (B) tensor of sigma value for the conditional input augmentation + condition_video_pose: (B, 1, T, H, W) tensor of pose condition + """ + B, C, T, H, W = x.shape + + if data_type == DataType.VIDEO: + assert condition_video_input_mask is not None, ( + "condition_video_input_mask is required for video data type; check if your model_obj is extend_model.FSDPDiffusionModel or the base DiffusionModel" + ) + if self.cp_group is not None: + condition_video_input_mask = rearrange( + condition_video_input_mask, "B C (V T) H W -> B C V T H W", V=self.n_cameras + ) + condition_video_input_mask = split_inputs_cp( + condition_video_input_mask, seq_dim=3, cp_group=self.cp_group + ) + condition_video_input_mask = rearrange( + condition_video_input_mask, "B C V T H W -> B C (V T) H W", V=self.n_cameras + ) + input_list = [x, condition_video_input_mask] + if condition_video_pose is not None: + if condition_video_pose.shape[2] > T: + logging.warning( + f"condition_video_pose has more frames than the input video: {condition_video_pose.shape} > {x.shape}" + ) + condition_video_pose = condition_video_pose[:, :, :T, :, :].contiguous() + input_list.append(condition_video_pose) + x = torch.cat( + input_list, + dim=1, + ) + + return super().forward( + x=x, + timesteps=timesteps, + crossattn_emb=crossattn_emb, + condition_video_augment_sigma=condition_video_augment_sigma, + **kwargs, + ) + + +def dit_data_step_no_split_on_cp(module, dataloader_iter): + batch = next(dataloader_iter)[0] + + batch = {k: v.to(device="cuda", non_blocking=True) if torch.is_tensor(v) else v for k, v in batch.items()} + + batch["is_preprocessed"] = True # assume data is preprocessed + if ("seq_len_q" in batch) and ("seq_len_kv" in batch): + cu_seqlens = batch["seq_len_q"].cumsum(dim=0).to(torch.int32) + zero = torch.zeros(1, dtype=torch.int32, device="cuda") + cu_seqlens = torch.cat((zero, cu_seqlens)) + + cu_seqlens_kv = batch["seq_len_kv"].cumsum(dim=0).to(torch.int32) + cu_seqlens_kv = torch.cat((zero, cu_seqlens_kv)) + + batch["packed_seq_params"] = { + "self_attention": PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + qkv_format=module.qkv_format, + ), + "cross_attention": PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens_kv, + qkv_format=module.qkv_format, + ), + } + + return batch + + +@dataclass +class MultiCameraDiT7BConfig(DiT7BConfig): + n_cameras: int = 4 + camera_condition_dim: int = 4 + traj_condition_dim: int = 0 + add_repeat_frame_embedding: bool = True + concat_camera_embedding: bool = True + concat_traj_embedding: bool = False + pixel_chunk_duration: int = 57 + data_step_fn = dit_data_step_no_split_on_cp + + @override + def configure_model(self, tokenizer=None) -> MultiCameraDiTCrossAttentionModel7B: + return MultiCameraDiTCrossAttentionModel7B( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + n_cameras=self.n_cameras, + camera_condition_dim=self.camera_condition_dim, + traj_condition_dim=self.traj_condition_dim, + add_repeat_frame_embedding=self.add_repeat_frame_embedding, + concat_camera_embedding=self.concat_camera_embedding, + concat_traj_embedding=self.concat_traj_embedding, + ) + + def configure_vae(self): + return dynamic_import(self.vae_module)(self.vae_path, pixel_chunk_duration=self.pixel_chunk_duration) + + +@dataclass +class VideoExtendMultiCameraDiT7BConfig(MultiCameraDiT7BConfig): + model_name = ("cosmos_7b_video2world",) + + @override + def configure_model(self, tokenizer=None) -> VideoExtendMultiCameraDiTCrossAttentionModel7B: + return VideoExtendMultiCameraDiTCrossAttentionModel7B( + self, + fp16_lm_cross_entropy=self.fp16_lm_cross_entropy, + parallel_output=self.parallel_output, + pre_process=parallel_state.is_pipeline_first_stage(), + post_process=parallel_state.is_pipeline_last_stage(), + n_cameras=self.n_cameras, + camera_condition_dim=self.camera_condition_dim, + traj_condition_dim=self.traj_condition_dim, + add_repeat_frame_embedding=self.add_repeat_frame_embedding, + concat_camera_embedding=self.concat_camera_embedding, + concat_traj_embedding=self.concat_traj_embedding, + ) + + +class FrameRepeatAttr(TrainingOnlyEmbModel): + def __init__(self): + super().__init__() + + def forward(self, frame_repeat: torch.Tensor) -> Dict[str, torch.Tensor]: + return { + "frame_repeat": frame_repeat / 10.0, + } + + def details(self) -> str: + return "Frame repeat, Output key: [frame_repeat]" + + +@attrs.define(slots=False) +class FrameRepeatConfig: + obj: Any = FrameRepeatAttr() # No arguments + dropout_rate: float = 0.0 + input_key: str = "frame_repeat" + + +class TrajectoryAttr(AbstractEmbModel): + def __init__(self, traj_dim: int): + super().__init__() + self.traj_dim = traj_dim + + def forward(self, traj: torch.Tensor) -> Dict[str, torch.Tensor]: + return { + "trajectory": traj, + } + + def details(self) -> str: + return f"Traj dim : {self.traj_dim} \n\tOutput key: [trajectory]" + + +@attrs.define(slots=False) +class TrajectoryConfig: + # context_dim == t5_text_embeddings_dim + obj: Any = TrajectoryAttr(traj_dim=192) + dropout_rate: float = 0.2 + input_key: str = "trajectory" + + +class MultiCamCosmosDiffusionPipeline(CosmosDiffusionPipeline): + def __init__(self, n_cameras, *args, **kwargs): + super().__init__(*args, **kwargs) + self.n_cameras = n_cameras + + # @torch.no_grad() + # def encode(self, state: torch.Tensor) -> torch.Tensor: + # state = rearrange(state, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + # encoded_state = self.vae.encode(state) + # encoded_state = rearrange(encoded_state, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) * self.sigma_data + # return encoded_state + + # @torch.no_grad() + # def decode(self, latent: torch.Tensor) -> torch.Tensor: + # latent = rearrange(latent, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + # decoded_state = self.vae.decode(latent / self.sigma_data) + # decoded_state = rearrange(decoded_state, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + # return decoded_state + + def compute_loss_with_epsilon_and_sigma( + self, + data_batch: dict[str, torch.Tensor], + x0_from_data_batch: torch.Tensor, + x0: torch.Tensor, + condition: Edify4Condition, + epsilon: torch.Tensor, + sigma: torch.Tensor, + ): + # Only support video batch + if parallel_state.is_initialized(): + if parallel_state.get_context_parallel_world_size() > 1: + # Turn on CP + cp_group = parallel_state.get_context_parallel_group() + + x0 = rearrange(x0, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + epsilon = rearrange(epsilon, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + x0 = split_inputs_cp(x=x0, seq_dim=2, cp_group=cp_group) + epsilon = split_inputs_cp(x=epsilon, seq_dim=2, cp_group=cp_group) + + x0 = rearrange(x0, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + epsilon = rearrange(epsilon, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + + output_batch, kendall_loss, pred_mse, edm_loss = super().compute_loss_with_epsilon_and_sigma( + data_batch, x0_from_data_batch, x0, condition, epsilon, sigma + ) + if not self.is_image_batch(data_batch): + if self.loss_reduce == "sum" and parallel_state.get_context_parallel_world_size() > 1: + kendall_loss *= parallel_state.get_context_parallel_world_size() + + return output_batch, kendall_loss, pred_mse, edm_loss + + def generate_samples_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + seed: int = 1, + state_shape: Tuple | None = None, + n_sample: int | None = None, + is_negative_prompt: bool = False, + num_steps: int = 35, + solver_option: COMMON_SOLVER_OPTIONS = "2ab", + ) -> Tensor: + """ + Generate samples from the batch. Based on given batch, it will automatically determine whether to generate image or video samples. + """ + + is_image_batch = self.is_image_batch(data_batch) + if n_sample is None: + input_key = self.input_image_key if is_image_batch else self.input_data_key + n_sample = data_batch[input_key].shape[0] + if state_shape is None: + if is_image_batch: + state_shape = (self.state_shape[0], 1, *self.state_shape[2:]) # C,T,H,W + + cp_enabled = parallel_state.get_context_parallel_world_size() > 1 + + if self._noise_generator is None: + self._initialize_generators() + + x0_fn = self.get_x0_fn_from_batch(data_batch, guidance, is_negative_prompt=is_negative_prompt) + + state_shape = list(state_shape) + + if len(state_shape) == 4: + state_shape = [1] + state_shape + np.random.seed(self.seed) + x_sigma_max = ( + torch.from_numpy(np.random.randn(*state_shape).astype(np.float32)).to( + dtype=torch.float32, device=self.tensor_kwargs["device"] + ) + * self.sde.sigma_max + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + x_sigma_max = rearrange(x_sigma_max, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + x_sigma_max = split_inputs_cp(x=x_sigma_max, seq_dim=2, cp_group=cp_group) + x_sigma_max = rearrange(x_sigma_max, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + + if self.sampler_type == "EDM": + samples = self.sampler(x0_fn, x_sigma_max, num_steps=num_steps, sigma_max=self.sde.sigma_max) + elif self.sampler_type == "RES": + samples = self.sampler( + x0_fn, x_sigma_max, sigma_max=self.sde.sigma_max, num_steps=num_steps, solver_option=solver_option + ) + + if cp_enabled: + cp_group = parallel_state.get_context_parallel_group() + samples = rearrange(samples, "B C (V T) H W -> (B V) C T H W", V=self.n_cameras) + samples = cat_outputs_cp(samples, seq_dim=2, cp_group=cp_group) + samples = rearrange(samples, "(B V) C T H W -> B C (V T) H W", V=self.n_cameras) + + return samples + + def get_x0_fn_from_batch( + self, + data_batch: Dict, + guidance: float = 1.5, + is_negative_prompt: bool = False, + ) -> Callable: + """ + Generates a callable function `x0_fn` based on the provided data batch and guidance factor. + + This function first processes the input data batch through a conditioning workflow (`conditioner`) to obtain conditioned and unconditioned states. It then defines a nested function `x0_fn` which applies a denoising operation on an input `noise_x` at a given noise level `sigma` using both the conditioned and unconditioned states. + + Args: + - data_batch (Dict): A batch of data used for conditioning. The format and content of this dictionary should align with the expectations of the `self.conditioner` + - guidance (float, optional): A scalar value that modulates the influence of the conditioned state relative to the unconditioned state in the output. Defaults to 1.5. + - is_negative_prompt (bool): use negative prompt t5 in uncondition if true + + Returns: + - Callable: A function `x0_fn(noise_x, sigma)` that takes two arguments, `noise_x` and `sigma`, and return x0 predictoin + + The returned function is suitable for use in scenarios where a denoised state is required based on both conditioned and unconditioned inputs, with an adjustable level of guidance influence. + """ + if is_negative_prompt: + condition, uncondition = self.conditioner.get_condition_with_negative_prompt(data_batch) + else: + condition, uncondition = self.conditioner.get_condition_uncondition(data_batch) + + # @TODO(tylerz): Why don't we need to broadcast the condition and uncondition tensors here? + # to_cp = parallel_state.get_context_parallel_world_size() > 1 + + # # For inference, check if parallel_state is initialized + # if parallel_state.is_initialized(): + # condition = broadcast_condition(condition, to_tp=True, to_cp=to_cp) + # uncondition = broadcast_condition(uncondition, to_tp=True, to_cp=to_cp) + # else: + # assert not to_cp, "parallel_state is not initialized, context parallel should be turned off." + + def x0_fn(noise_x: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor: + cond_x0, _, _ = self.denoise(noise_x, sigma, condition) + uncond_x0, _, _ = self.denoise(noise_x, sigma, uncondition) + return cond_x0 + guidance * (cond_x0 - uncond_x0) + + return x0_fn + + +class MultiCameraDiTModel(DiTModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.vae = None + self.conditioner = VideoConditioner( + text=TextConfig(), + fps=FPSConfig(), + num_frames=NumFramesConfig(), + image_size=ImageSizeConfig(), + padding_mask=PaddingMaskConfig(), + frame_repeat=FrameRepeatConfig(), + trajectory=TrajectoryConfig(), + ) + self.diffusion_pipeline = MultiCamCosmosDiffusionPipeline( + net=self, + n_cameras=self.config.n_cameras, + conditioner=self.conditioner, + loss_add_logvar=self.config.loss_add_logvar, + ) + + @torch.no_grad() + def encode(self, state: torch.Tensor) -> torch.Tensor: + state = rearrange(state, "B C (V T) H W -> (B V) C T H W", V=self.config.n_cameras) + encoded_state = self.vae.encode(state) + encoded_state = rearrange(encoded_state, "(B V) C T H W -> B C (V T) H W", V=self.config.n_cameras) + return encoded_state + + @torch.no_grad() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + latent = rearrange(latent, "B C (V T) H W -> (B V) C T H W", V=self.config.n_cameras) + decoded_state = self.vae.decode(latent / self.config.sigma_data) + decoded_state = rearrange(decoded_state, "(B V) C T H W -> B C (V T) H W", V=self.config.n_cameras) + return decoded_state + + @torch.no_grad() + def data_step(self, dataloader_iter): + batch = super().data_step(dataloader_iter) + batch = {k: v.to(dtype=torch.bfloat16) if torch.is_tensor(v) else v for k, v in batch.items()} + + if self.vae is None: + self.vae = self.config.configure_vae() + self.vae.to("cuda") + + batch["video"] = self.encode(batch["video"]) + seq_len = batch["video"].shape[-1] * batch["video"].shape[-2] * batch["video"].shape[-3] + batch["loss_mask"] = torch.ones(seq_len, dtype=torch.bfloat16, device=batch["video"].device) + from megatron.core import mpu + + cp_size = mpu.get_context_parallel_world_size() + cp_rank = mpu.get_context_parallel_rank() + + if cp_size > 1: + num_valid_tokens_in_ub = None + if "loss_mask" in batch and batch["loss_mask"] is not None: + num_valid_tokens_in_ub = batch["loss_mask"].sum() + batch["num_valid_tokens_in_ub"] = num_valid_tokens_in_ub + batch["loss_mask"] = ( + batch["loss_mask"].view(cp_size, batch["loss_mask"].shape[0] // cp_size)[cp_rank, ...].contiguous() + ) + + return batch + + def validation_step(self, batch, batch_idx=None) -> torch.Tensor: + # In mcore the loss-function is part of the forward-pass (when labels are provided) + state_shape = batch["video"].shape + sample = self.diffusion_pipeline.generate_samples_from_batch( + batch, + guidance=7, + state_shape=state_shape, + num_steps=35, + is_negative_prompt=True if "neg_t5_text_embeddings" in batch else False, + ) + + video = (1.0 + self.decode(sample)).clamp(0, 2) / 2 # [B, 3, T, H, W] + + video = (video * 255).to(torch.uint8).cpu().numpy().astype(np.uint8) + + video_segments = einops.rearrange(video, "b c (v t) h w -> b c v t h w", v=self.config.n_cameras) + if self.config.n_cameras == 4: + grid_video = einops.rearrange(video_segments, "b c (h w) t h1 w1 -> b c t (h h1) (w w1)", h=2, w=2) + elif self.config.n_cameras == 6: + grid_video = torch.stack( + [ + torch.from_numpy(video_segments[:, :, 1]), + torch.from_numpy(video_segments[:, :, 0]), + torch.from_numpy(video_segments[:, :, 2]), + torch.from_numpy(video_segments[:, :, 4]), + torch.from_numpy(video_segments[:, :, 3]), + torch.from_numpy(video_segments[:, :, 5]), + ], + dim=2, + ) + grid_video = einops.rearrange(grid_video, "b c (h w) t h1 w1 -> b c t (h h1) (w w1)", h=2, w=3) + + grid_video = rearrange(grid_video, "b c t h w -> b t c h w") + + # wandb is on the last rank for megatron, first rank for nemo + wandb_rank = 0 + + if parallel_state.get_data_parallel_src_rank() == wandb_rank: + if torch.distributed.get_rank() == wandb_rank: + gather_list = [None for _ in range(parallel_state.get_data_parallel_world_size())] + else: + gather_list = None + torch.distributed.gather_object( + grid_video, gather_list, wandb_rank, group=parallel_state.get_data_parallel_group() + ) + if gather_list is not None: + videos = [] + for grid_video in gather_list: + grid_video = grid_video.numpy() + videos.append(wandb.Video(grid_video, fps=12)) + wandb.log({"prediction": videos}, step=self.global_step) + + return None + + def on_validation_end(self): + pass diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/multicamera.py b/nemo_vfm/physicalai/diffusion/post_training/multicamera/multicamera.py new file mode 100644 index 00000000..659aaeed --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/multicamera.py @@ -0,0 +1,245 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import glob +import os +from functools import partial + +import nemo_run as run +from huggingface_hub import snapshot_download +from lightning.pytorch.utilities.types import EVAL_DATALOADERS, TRAIN_DATALOADERS +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.train import pretrain +from nemo.collections.llm.gpt.data.mock import MockDataModule +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.alpamayo_dataloader import ( + InfiniteDataVerse, +) +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.config_dataverse import DATAVERSE_CONFIG +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.dataloader_utils import ( + dict_collation_fn, +) +from nemo.collections.physicalai.diffusion.post_training.multicamera.dit_multi_camera import ( + MultiCameraDiT7BConfig, + MultiCameraDiTModel, +) +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.strategies.utils import RestoreConfig +from torch.utils.data import DataLoader + + +class SimpleDataModule(MockDataModule): + def __init__(self, *args, dataset=None, **kwargs): + super().__init__(*args, **kwargs) + self.dataset = dataset + + def setup(self, stage: str = "") -> None: + self._train_ds = self.dataset() + self._validation_ds = self.dataset() + self._test_ds = self.dataset() + + def train_dataloader(self) -> TRAIN_DATALOADERS: + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds, num_workers=8) + + def val_dataloader(self) -> EVAL_DATALOADERS: + if not hasattr(self, "_validation_ds"): + self.setup() + return self._create_dataloader(self._validation_ds, num_workers=0) + + def _create_dataloader(self, dataset, num_workers=0, **kwargs) -> DataLoader: + return DataLoader( + dataset, + num_workers=num_workers, + pin_memory=self.pin_memory, + persistent_workers=self.persistent_workers, + collate_fn=dict_collation_fn, + **kwargs, + ) + + +def get_latest_checkpoint(checkpoint_dir): + # Get all checkpoint files + checkpoint_files = glob.glob(os.path.join(checkpoint_dir, "*.ckpt")) + + if not checkpoint_files: + return None # No checkpoint found + + # Sort by modification time (latest first) + latest_ckpt = max(checkpoint_files, key=os.path.getmtime) + return latest_ckpt + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model = run.Config( + MultiCameraDiTModel, + config=run.Config( + MultiCameraDiT7BConfig, + n_cameras=6, + camera_condition_dim=6, + add_repeat_frame_embedding=True, + # concat_traj_embedding=True, + # traj_condition_dim=12, + vae_path=snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8"), + pixel_chunk_duration=57, + recompute_granularity="full", + recompute_method="uniform", + recompute_num_layers=1, + ), + ) + recipe.trainer.strategy.ckpt_load_strictness = False + recipe.trainer.val_check_interval = 100 + recipe.trainer.limit_val_batches = 1 + + # Trainer setup + recipe.trainer.max_steps = 30000 + + # Optim setup + recipe.optim.config.lr = 1e-4 + recipe.optim.config.weight_decay = 0.3 + recipe.optim.config.adam_eps = 1e-8 + recipe.optim.config.adam_beta1 = 0.9 + recipe.optim.config.adam_beta2 = 0.999 + recipe.optim.lr_scheduler = run.Config( + nl.lr_scheduler.WarmupHoldPolicyScheduler, warmup_steps=1000, min_lr=1.0e-6, hold_steps=1e9 + ) + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 4 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + recipe.trainer.strategy.save_ckpt_format = "torch_dist" + + # FSDP + # recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + # recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = False + recipe.trainer.strategy.ddp.overlap_grad_reduce = False + recipe.model.config.use_cpu_initialization = True + + recipe.trainer.callbacks = [ + run.Config( + ModelCheckpoint, + monitor="reduced_train_loss", + dirpath="nemo_experiments/cosmos_multicamera_diffusion_7b_text2world_finetune/default/experiment_dir", + filename="{epoch}-{step}", + every_n_train_steps=100, + save_top_k=5, + always_save_context=True, + save_weights_only=False, + ), + run.Config(PreemptionCallback), + ] + + # Data setup + recipe.data = run.Config( + SimpleDataModule, + micro_batch_size=1, + global_batch_size=32, + dataset=partial(InfiniteDataVerse, **DATAVERSE_CONFIG["alpamayo_v2_traj_qwen_24fps_6_cameras_frame_repeat"]), + ) + + recipe.resume = run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + resume_from_directory="nemo_experiments/cosmos_multicamera_diffusion_7b_text2world_finetune/default/experiment_dir", + ) + + # Checkpoint load + recipe.resume.restore_config = run.Config( + RestoreConfig, + path=os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Text2World", allow_patterns=["nemo/*"]), "nemo" + ), # path to diffusion model checkpoint + load_model_state=True, + load_optim_state=True, + load_artifacts=False, + ) + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune_w_traj() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + recipe.model.config.concat_traj_embedding = True + recipe.model.config.traj_condition_dim = 12 + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune_w_traj_debug() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune_w_traj() + + recipe.model.config.concat_traj_embedding = True + recipe.model.config.traj_condition_dim = 12 + recipe.model.config.num_layers = 1 + recipe.resume.restore_config = None + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_image2world_finetune() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + recipe.model = run.Config( + MultiCameraDiTModel, + config=run.Config( + MultiCameraDiT7BConfig, + n_cameras=6, + camera_condition_dim=6, + add_repeat_frame_embedding=True, + # concat_traj_embedding=True, + # traj_condition_dim=12, + vae_path=snapshot_download("nvidia/Cosmos-1.0-Tokenizer-CV8x8x8"), + pixel_chunk_duration=57, + # recompute_granularity="full", + # recompute_method="uniform", + # recompute_num_layers=1, + ), + ) + + # Checkpoint load + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + return recipe + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_image2world_finetune_w_traj() -> run.Partial: + # Model setup + recipe = cosmos_multicamera_diffusion_7b_image2world_finetune() + + recipe.model.config.concat_traj_embedding = True + recipe.model.config.traj_condition_dim = 32 + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=cosmos_multicamera_diffusion_7b_text2world_finetune) diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/multicamera_mock.py b/nemo_vfm/physicalai/diffusion/post_training/multicamera/multicamera_mock.py new file mode 100644 index 00000000..cf0d22bb --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/multicamera_mock.py @@ -0,0 +1,131 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +import torch +from huggingface_hub import snapshot_download +from nemo.collections import llm +from nemo.collections.diffusion.datamodule import DiTDataModule +from nemo.collections.diffusion.train import pretrain +from nemo.collections.physicalai.diffusion.post_training.multicamera.dit_multi_camera import MultiCameraDiT7BConfig +from nemo.lightning.pytorch.strategies.utils import RestoreConfig + + +class MultiCameraDiTVideoLatentMockDataset(torch.utils.data.Dataset): + def __init__(self, num_samples, seq_len=21760): + self.length = num_samples if num_samples > 0 else 1 << 32 + self.seq_len = seq_len + + def __len__(self): + return self.length + + def __getitem__(self, idx): + t = 16 + h = 34 + w = 40 + seq_len = t * h * w + video_latent = torch.randn(16, t, h, w).to(dtype=torch.uint8) + loss_mask = torch.ones(seq_len, dtype=torch.bfloat16) + noise_latent = torch.rand_like(video_latent, dtype=torch.bfloat16) + timesteps = torch.randn(1, dtype=torch.bfloat16) + text_embedding = torch.randn(512, 1024, dtype=torch.bfloat16) + + return { + "video": video_latent, + "noise_latent": noise_latent, + "timesteps": timesteps, + "t5_text_embeddings": text_embedding, + "t5_text_mask": torch.ones(512, dtype=torch.bfloat16), + "image_size": torch.tensor([[34, 40, 34, 40]] * 1, dtype=torch.bfloat16), + "fps": torch.tensor([30] * 1, dtype=torch.bfloat16), + "num_frames": torch.tensor([16] * 1, dtype=torch.bfloat16), + "padding_mask": torch.zeros((1, 1, 34, 40), dtype=torch.bfloat16), + "loss_mask": loss_mask, + } + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +@run.cli.factory(target=llm.train) +def cosmos_multicamera_diffusion_7b_text2world_finetune() -> run.Partial: + # Model setup + recipe = pretrain() + recipe.model.config = run.Config(MultiCameraDiT7BConfig) + recipe.trainer.strategy.ckpt_load_strictness = False + + # Trainer setup + recipe.trainer.max_steps = 1000 + recipe.optim.config.lr = 1e-6 + + # Tensor / Sequence parallelism + recipe.trainer.strategy.tensor_model_parallel_size = 8 + recipe.trainer.strategy.sequence_parallel = True + recipe.trainer.strategy.ckpt_async_save = False + + # FSDP + recipe.trainer.strategy.ddp.with_megatron_fsdp_code_path = True + recipe.trainer.strategy.ddp.data_parallel_sharding_strategy = "MODEL_AND_OPTIMIZER_STATES" + recipe.trainer.strategy.ddp.overlap_param_gather = True + recipe.trainer.strategy.ddp.overlap_grad_reduce = True + recipe.model.config.use_cpu_initialization = True + + # Data setup + recipe.data = DiTDataModule( + dataset=MultiCameraDiTVideoLatentMockDataset, path=1000, micro_batch_size=1, global_batch_size=1 + ) + + # Checkpoint load + recipe.resume.restore_config = run.Config(RestoreConfig, load_artifacts=False) + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Text2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + recipe.resume.resume_if_exists = False + + # Directory to save checkpoints / logs + recipe.log.log_dir = "nemo_experiments/cosmos_multicamera_diffusion_7b_text2world_finetune" + + return recipe + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=cosmos_multicamera_diffusion_7b_text2world_finetune) diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/test_dataloader.py b/nemo_vfm/physicalai/diffusion/post_training/multicamera/test_dataloader.py new file mode 100644 index 00000000..b28d9aad --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/test_dataloader.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo.collections.physicalai.datasets.dataverse_dataset.driving_dataloader.alpamayo_dataloader import ( + DrivingVideoDataLoader, + get_driving_dataset, +) + + +if __name__ == "__main__": + d = DrivingVideoDataLoader( + dataset=get_driving_dataset( + dataset_name="alpamayo_v2_traj_qwen_24fps_6_cameras_frame_repeat", + ), + batch_size=1, + num_workers=6, + shuffle=True, + prefetch_factor=4, + ) + tmp = next(iter(d)) + from IPython import embed + + embed() diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/train.sh b/nemo_vfm/physicalai/diffusion/post_training/multicamera/train.sh new file mode 100644 index 00000000..31bbd09e --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/train.sh @@ -0,0 +1,35 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +# example slurm script for training diffusion + +#SBATCH -p your_partition -A your_account -t 24:00:00 --nodes=16 --exclusive --mem=0 --overcommit --gpus-per-node 8 --ntasks-per-node=8 --dependency=singleton + +export HF_TOKEN=xxx +export WANDB_API_KEY=xxx +export WANDB_PROJECT=xxx +export WANDB_RESUME=allow +export NVTE_FUSED_ATTN=0 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export HF_HOME=xxx +export XDG_CACHE_HOME=xxx + +DIR=`pwd` + +srun -l --container-image nvcr.io#nvidia/nemo:cosmos.1.0.1 --container-mounts "/lustre:/lustre/,/home:/home" --no-container-mount-home --mpi=pmix bash -c ' +cd $DIR; +PYTHONPATH=.:nemo/collections/physicalai/datasets/dataverse:$PYTHONPATH; +python -u nemo/collections/physicalai/diffusion/post_training/multicamera/multicamera.py --yes' \ No newline at end of file diff --git a/nemo_vfm/physicalai/diffusion/post_training/multicamera/validate_multicamera.py b/nemo_vfm/physicalai/diffusion/post_training/multicamera/validate_multicamera.py new file mode 100644 index 00000000..488b739b --- /dev/null +++ b/nemo_vfm/physicalai/diffusion/post_training/multicamera/validate_multicamera.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +from huggingface_hub import snapshot_download +from multicamera import cosmos_multicamera_diffusion_7b_text2world_finetune +from nemo.collections import llm + + +@run.cli.factory(target=llm.validate) +def cosmos_multicamera_diffusion_7b_text2world_validate() -> run.Partial: + recipe = cosmos_multicamera_diffusion_7b_text2world_finetune() + + # Checkpoint load + recipe.resume.restore_config.path = os.path.join( + snapshot_download("nvidia/Cosmos-1.0-Diffusion-7B-Video2World", allow_patterns=["nemo/*"]), "nemo" + ) # path to diffusion model checkpoint + + return run.Partial( + llm.validate, + model=recipe.model, + data=recipe.data, + trainer=recipe.trainer, + log=recipe.log, + optim=recipe.optim, + tokenizer=None, + resume=recipe.resume, + model_transform=None, + ) + + +if __name__ == "__main__": + run.cli.main(llm.validate, default_factory=cosmos_multicamera_diffusion_7b_text2world_validate) diff --git a/nemo_vfm/physicalai/tokenizer/README.md b/nemo_vfm/physicalai/tokenizer/README.md new file mode 100644 index 00000000..52c54ac3 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/README.md @@ -0,0 +1,132 @@ +# Cosmos Tokenizer: NeMo Framework Finetuning User Guide + +Fine-tuning the Cosmos tokenizer using the [NVIDIA NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html) enables it to more accurately model previously unseen scenarios in customer data, particularly in the context of self-driving applications. By adapting the tokenizer to the specific characteristics and complexities of in-house video content, it becomes better equipped to handle unique visual and temporal patterns that may not have been captured during its initial pre-training. This enhanced modeling capability is critical for downstream diffusion models, which rely on the tokenizer's output to generate realistic physical scenes, ultimately improving the performance and safety of self-driving car systems. + +## Model Support Matrix + +The NeMo Framework supports the following Cosmos Tokenizer models. Review the available models for post-training. + +| Model Name | Model Ckpt | +|-------------------------|----------------------------| +| Cosmos-1.0-Tokenizer-CV8x8x8 | [HF Download](https://huggingface.co/nvidia/Cosmos-1.0-Tokenizer-CV8x8x8) | +| Cosmos-1.0-Tokenizer-DV8x16x16 | [HF Download](https://huggingface.co/nvidia/Cosmos-1.0-Tokenizer-DV8x16x16) | + +If you have a specific use case that would benefit from alternative tokenizers, please do not hesitate to submit a request. For optimal performance, we recommend utilizing GPUs such as the H100-80GB or A100-80GB. + +## Post-Training Support Matrix + +Cosmos tokenizer can be post-trained for a variety of Physical AI tasks. Review the following table for a list of available Physical AI post-training tasks: + +| Post-training Task | Support Status | +|-------------------------|--------------------| +| General post-training and validation | **Supported** | + +## Prerequisites + +### 1. Review General Requirements + +- System Configuration + - **NVIDIA GPU and driver**: Ensure you have access to the 80G H100 or A100 to run the model(s). + - **Containerization Platform**: We recommend using NVIDIA [NeMo Docker](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo/tags) Runtime (alternatively, you may use NVIDIA enroot). +- Get your [Hugging Face User Access Token](https://huggingface.co/docs/hub/en/security-tokens), which is required to obtain the Cosmos models for training and inference. +- Get your [Weights and Biases API Key](https://docs.wandb.ai/support/find_api_key/) for logging and tracking. + +### 2. Clone the Cosmos Repository + +```bash +git clone git@github.com:NVIDIA/Cosmos.git +``` + +### 3. Start the Container + +The [NeMo Framework container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nemo) supports post-training and inference for Cosmos Tokenizer models. + +Run the following command to download and start the container: + ```bash + docker run --ipc=host -it --gpus=all \ + -v $PATH_TO_COSMOS_REPO:/workspace/Cosmos \ + nvcr.io/nvidia/nemo:24.12.01 bash + ``` + +### 4. Download Checkpoints + +Please refer to the link provided in the Model Support Matrix to download the Cosmos tokenizer checkpoints from HuggingFace. Detailed instructions for the download process are available on the HuggingFace page. + +## Post-train + +Finetuning a Cosmos tokenizer enables you to train the model to compress videos that are more specific to your Physical AI use case. + +There are 3 steps to finetuning: preparing a dataset, preprocessing the data, and post-training the model. + +### 1. Prepare a Dataset + +The first step is to prepare your dataset. Organize your data into a folder containing multiple video tars, each contains MP4 format videos (preferably at least 720p resolution). The recommended folder structure is as follows: + +- **000000.tar** + - `1.mp4` + - `2.mp4` +- **000001.tar** + - `3.mp4` + - `4.mp4` + +Here, 000000.tar and 000001.tar represent separate shards, and you may include additional shards as needed. + +### 3. Post-train the Model + +The third step is to fine-tune the Cosmos tokenizer using the NeMo Framework. + +#### Run the Post-training Script + +Complete the following steps to finetune the Cosmos tokenizer Cosmos-1.0-Tokenizer-CV8x8x8. + +1. Install the dependencies under cosmos1/models/tokenizer/nemo: + ```bash + pip install megatron-energon==4.0.0 pyav + pip install git+https://github.com/NVIDIA/NeMo-Run.git + pip install moviepy==1.0.3 imageio + ``` +2. Install Cosmos project to get the tokenizer model: + ```bash + git clone https://github.com/NVIDIA/Cosmos.git + cd Cosmos + pip install --no-deps . + ``` +3. Run the following command for finetuning Cosmos-1.0-Tokenizer-CV8x8x8: + ```bash + export CKPT_PTH="" + export DATA="" + + # Optionally, you can monitor training progress with Weights and Biases (wandb). + export WANDB_API_KEY="
" + export WANDB_PROJECT_NAME="cosmos-diffusion-nemo-post-training" + export WANDB_RUN_ID="cosmos_diffusion_7b_text2world_finetune" + + torchrun --nproc-per-node 8 cosmos1/models/tokenizer/nemo/train_tokenizer.py --yes \ + data.path=$DATA \ + model.jit_ckpt_pth=$CKPT_PTH \ + model.model="Cosmos-1.0-Tokenizer-CV8x8x8" + ``` + +##### Configurable Hyperparameters + +For a comprehensive list of configurable hyperparameters, please refer to the `train_tokenizer.py` script. The script supports four major configuration components: + +1. **model** +2. **data** +3. **trainer** +4. **optim** + +You can configure any hyperparameter of these four components by setting the value in the launch script using the following format: + +```bash +model.jit_ckpt_pth= +trainer.max_epochs= +``` + +Adjust the values as needed to suit your training requirements. After a few hundred iterations, you should observe that the 'loss' reported in Weights & Biases (wandb) starts decreasing. + +Below is an example of loss curve after a few hundreds iteration of finetuning + +

+ Image description +

diff --git a/nemo_vfm/physicalai/tokenizer/assets/loss.png b/nemo_vfm/physicalai/tokenizer/assets/loss.png new file mode 100644 index 00000000..aa01c9f3 Binary files /dev/null and b/nemo_vfm/physicalai/tokenizer/assets/loss.png differ diff --git a/nemo_vfm/physicalai/tokenizer/augmentors.py b/nemo_vfm/physicalai/tokenizer/augmentors.py new file mode 100644 index 00000000..ea47bf1d --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/augmentors.py @@ -0,0 +1,182 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + + +"""Additional augmentors for image and video training loops.""" + +import logging +import random +from typing import Optional + +import torch +from nemo.collections.physicalai.tokenizer.data.augmentors.augmentor import Augmentor +from nemo.collections.physicalai.tokenizer.data.augmentors.image.cropping import RandomCrop +from nemo.collections.physicalai.tokenizer.data.augmentors.image.misc import ( + obtain_augmentation_size, + obtain_image_size, +) +from nemo.collections.physicalai.tokenizer.data.augmentors.image.resize import ResizeSmallestSideAspectPreserving + + +class LossMask(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs data normalization. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are center cropped. + """ + assert self.args is not None, "Please specify args" + mask_config = self.args["masking"] + + input_key = self.input_keys[0] + default_mask = torch.ones_like(data_dict[input_key]) + loss_mask = mask_config["nonhuman_mask"] * default_mask + for curr_key in mask_config: + if curr_key not in self.input_keys: + continue + curr_mask = data_dict[curr_key] + curr_weight = mask_config[curr_key] + curr_loss_mask = curr_mask * curr_weight + (1 - curr_mask) * loss_mask + loss_mask = torch.max(curr_loss_mask, loss_mask) + _ = data_dict.pop(curr_key) + data_dict["loss_mask"] = loss_mask + return data_dict + + +class UnsqueezeImage(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs horizontal flipping. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are center cropped. + """ + for key in self.input_keys: + data_dict[key] = data_dict[key].unsqueeze(1) + + return data_dict + + +class RandomReverse(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs random temporal reversing of frames. + + Args: + data_dict (dict): Input data dict, CxTxHxW + Returns: + data_dict (dict): Output dict where videos are randomly reversed. + """ + assert self.args is not None + p = self.args.get("prob", 0.5) + coin_flip = torch.rand(1).item() <= p + for key in self.input_keys: + if coin_flip: + data_dict[key] = torch.flip(data_dict[key], dims=[1]) + + return data_dict + + +class RenameInputKeys(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Rename the input keys from the data dict. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict with keys renamed. + """ + assert len(self.input_keys) == len(self.output_keys) + for input_key, output_key in zip(self.input_keys, self.output_keys): + if input_key in data_dict: + data_dict[output_key] = data_dict.pop(input_key) + return data_dict + + +class CropResizeAugmentor(Augmentor): + def __init__( + self, + input_keys: list, + output_keys: Optional[list] = None, + crop_args: Optional[dict] = None, + resize_args: Optional[dict] = None, + args: Optional[dict] = None, + ) -> None: + super().__init__(input_keys, output_keys, args) + self.crop_args = crop_args + self.resize_args = resize_args + self.crop_op = RandomCrop(input_keys, output_keys, crop_args) + self.resize_op = ResizeSmallestSideAspectPreserving(input_keys, output_keys, resize_args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs random temporal reversing of frames. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where videso are randomly reversed. + """ + assert self.args is not None + p = self.args.get("prob", 0.5) + + if p > 0.0: + crop_img_size = obtain_augmentation_size(data_dict, self.crop_args) + crop_width, crop_height = crop_img_size + orig_w, orig_h = obtain_image_size(data_dict, self.input_keys) + if orig_w < crop_width or orig_h < crop_height: + logging.warning( + f"Data size ({orig_w}, {orig_h}) is smaller than crop size ({crop_width}, {crop_height}), skip the crop augmentation." + ) + coin_flip = torch.rand(1).item() <= p + if coin_flip and crop_width <= orig_w and crop_height <= orig_h: + data_dict = self.crop_op(data_dict) + return data_dict + + data_dict = self.resize_op(data_dict) + data_dict = self.crop_op(data_dict) + + return data_dict + + +class TemporalRandomCrop(object): + """Temporally crop the given frame indices at a random location. + + Args: + size (int): Desired length of frames will be seen in the model. + """ + + def __init__(self, size): + self.size = size + + def __call__(self, total_frames): + rand_end = max(0, total_frames - self.size - 1) + begin_index = random.randint(0, rand_end) + end_index = min(begin_index + self.size, total_frames) + return begin_index, end_index diff --git a/nemo_vfm/physicalai/tokenizer/configs.py b/nemo_vfm/physicalai/tokenizer/configs.py new file mode 100644 index 00000000..25be7493 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/configs.py @@ -0,0 +1,139 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from models import ContinuousFormulation, Decoder3DType, DecoderType, DiscreteQuantizer, Encoder3DType, EncoderType + + +continuous_image = dict( + # The attention resolution for res blocks. + attn_resolutions=[32], + # The base number of channels. + channels=128, + # The channel multipler for each resolution. + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + # The spatial compression ratio. + spatial_compression=16, + # The number of layers in each res block. + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + # The output latent dimension (channels). + latent_channels=16, + # The encoder output channels just before sampling. + # Which is also the decoder's input channels. + z_channels=16, + # A factor over the z_channels, to get the total channels the encoder should output. + # For a VAE for instance, we want to output the mean and variance, so we need 2 * z_channels. + z_factor=1, + name="CI", + # What formulation to use, either "AE" or "VAE". + # Chose VAE here, since the pre-trained ckpt were of a VAE formulation. + formulation=ContinuousFormulation.AE.name, + # Specify type of encoder ["Default", "LiteVAE"] + encoder=EncoderType.Default.name, + # Specify type of decoder ["Default"] + decoder=DecoderType.Default.name, +) + +discrete_image = dict( + # The attention resolution for res blocks. + attn_resolutions=[32], + # The base number of channels. + channels=128, + # The channel multipler for each resolution. + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + # The spatial compression ratio. + spatial_compression=16, + # The number of layers in each res block. + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + # The encoder output channels just before sampling. + z_channels=256, + # A factor over the z_channels, to get the total channels the encoder should output. + # for discrete tokenization, often we directly use the vector, so z_factor=1. + z_factor=1, + # The quantizer of choice, VQ, LFQ, FSQ, or ResFSQ. + quantizer=DiscreteQuantizer.FSQ.name, + # The embedding dimension post-quantization, which is also the input channels of the decoder. + # Which is also the output + embedding_dim=6, + # The number of levels to use for fine-scalar quantization. + levels=[8, 8, 8, 5, 5, 5], + # The number of quantizers to use for residual fine-scalar quantization. + num_quantizers=4, + name="DI", + # Specify type of encoder ["Default", "LiteVAE"] + encoder=EncoderType.Default.name, + # Specify type of decoder ["Default"] + decoder=DecoderType.Default.name, +) + +continuous_video = dict( + attn_resolutions=[32], + channels=128, + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + latent_channels=16, + z_channels=16, + z_factor=1, + num_groups=1, + legacy_mode=False, + spatial_compression=8, + temporal_compression=8, + formulation=ContinuousFormulation.AE.name, + encoder=Encoder3DType.FACTORIZED.name, + decoder=Decoder3DType.FACTORIZED.name, + name="CV", +) + +discrete_video = dict( + attn_resolutions=[32], + channels=128, + channels_mult=[2, 4, 4], + dropout=0.0, + in_channels=3, + num_res_blocks=2, + out_channels=3, + resolution=1024, + patch_size=4, + patch_method="haar", + z_channels=16, + z_factor=1, + num_groups=1, + legacy_mode=False, + spatial_compression=16, + temporal_compression=8, + quantizer=DiscreteQuantizer.FSQ.name, + embedding_dim=6, + levels=[8, 8, 8, 5, 5, 5], + encoder=Encoder3DType.FACTORIZED.name, + decoder=Decoder3DType.FACTORIZED.name, + name="DV", +) diff --git a/nemo_vfm/physicalai/tokenizer/data/__init__.py b/nemo_vfm/physicalai/tokenizer/data/__init__.py new file mode 100644 index 00000000..9e325007 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/__init__.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/__init__.py new file mode 100644 index 00000000..9e325007 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/augmentor.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/augmentor.py new file mode 100644 index 00000000..0de294d5 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/augmentor.py @@ -0,0 +1,34 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Any, Optional + + +class Augmentor: + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + r"""Base augmentor class + + Args: + input_keys (list): List of input keys + output_keys (list): List of output keys + args (dict): Arguments associated with the augmentation + """ + self.input_keys = input_keys + self.output_keys = output_keys + self.args = args + + def __call__(self, *args: Any, **kwds: Any) -> Any: + raise ValueError("Augmentor not implemented") diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/image/__init__.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/__init__.py new file mode 100644 index 00000000..9e325007 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/image/cropping.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/cropping.py new file mode 100644 index 00000000..271f4689 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/cropping.py @@ -0,0 +1,122 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import torch +import torchvision.transforms.functional as transforms_F +from loguru import logger as logging +from nemo.collections.physicalai.tokenizer.data.augmentors.augmentor import Augmentor +from nemo.collections.physicalai.tokenizer.data.augmentors.image.misc import ( + obtain_augmentation_size, + obtain_image_size, +) + + +class CenterCrop(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs center crop. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are center cropped. + We also save the cropping parameters in the aug_params dict + so that it will be used by other transforms. + """ + assert (self.args is not None) and ("size" in self.args), "Please specify size in args" + + img_size = obtain_augmentation_size(data_dict, self.args) + width, height = img_size + + orig_w, orig_h = obtain_image_size(data_dict, self.input_keys) + for key in self.input_keys: + data_dict[key] = transforms_F.center_crop(data_dict[key], [height, width]) + + # We also add the aug params we use. This will be useful for other transforms + crop_x0 = (orig_w - width) // 2 + crop_y0 = (orig_h - height) // 2 + cropping_params = { + "resize_w": orig_w, + "resize_h": orig_h, + "crop_x0": crop_x0, + "crop_y0": crop_y0, + "crop_w": width, + "crop_h": height, + } + + if "aug_params" not in data_dict: + data_dict["aug_params"] = dict() + + data_dict["aug_params"]["cropping"] = cropping_params + data_dict["padding_mask"] = torch.zeros((1, cropping_params["crop_h"], cropping_params["crop_w"])) + return data_dict + + +class RandomCrop(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs random crop. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are center cropped. + We also save the cropping parameters in the aug_params dict + so that it will be used by other transforms. + """ + + img_size = obtain_augmentation_size(data_dict, self.args) + width, height = img_size + + orig_w, orig_h = obtain_image_size(data_dict, self.input_keys) + # Obtaining random crop coords + try: + crop_x0 = int(torch.randint(0, orig_w - width + 1, size=(1,)).item()) + crop_y0 = int(torch.randint(0, orig_h - height + 1, size=(1,)).item()) + except Exception: + logging.warning( + f"Random crop failed. Performing center crop, original_size(wxh): {orig_w}x{orig_h}, random_size(wxh): {width}x{height}" + ) + for key in self.input_keys: + data_dict[key] = transforms_F.center_crop(data_dict[key], [height, width]) + crop_x0 = (orig_w - width) // 2 + crop_y0 = (orig_h - height) // 2 + + # We also add the aug params we use. This will be useful for other transforms + cropping_params = { + "resize_w": orig_w, + "resize_h": orig_h, + "crop_x0": crop_x0, + "crop_y0": crop_y0, + "crop_w": width, + "crop_h": height, + } + + if "aug_params" not in data_dict: + data_dict["aug_params"] = dict() + + data_dict["aug_params"]["cropping"] = cropping_params + + # We must perform same random cropping for all input keys + for key in self.input_keys: + data_dict[key] = transforms_F.crop(data_dict[key], crop_y0, crop_x0, height, width) + return data_dict diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/image/misc.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/misc.py new file mode 100644 index 00000000..29572b6b --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/misc.py @@ -0,0 +1,60 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Union + +import torch +from PIL import Image + + +def obtain_image_size(data_dict: dict, input_keys: list) -> tuple[int, int]: + r"""Function for obtaining the image size from the data dict. + + Args: + data_dict (dict): Input data dict + input_keys (list): List of input keys + Returns: + width (int): Width of the input image + height (int): Height of the input image + """ + + data1 = data_dict[input_keys[0]] + if isinstance(data1, Image.Image): + width, height = data1.size + elif isinstance(data1, torch.Tensor): + height, width = data1.size()[-2:] + else: + raise ValueError("data to random crop should be PIL Image or tensor") + + return width, height + + +def obtain_augmentation_size(data_dict: dict, augmentor_cfg: dict) -> Union[int, tuple]: + r"""Function for obtaining size of the augmentation. + When dealing with multi-aspect ratio dataloaders, we need to + find the augmentation size from the aspect ratio of the data. + + Args: + data_dict (dict): Input data dict + augmentor_cfg (dict): Augmentor config + Returns: + aug_size (int): Size of augmentation + """ + if "__url__" in data_dict and "aspect_ratio" in data_dict["__url__"].meta.opts: + aspect_ratio = data_dict["__url__"].meta.opts["aspect_ratio"] + aug_size = augmentor_cfg["size"][aspect_ratio] + else: # Non-webdataset format + aspect_ratio = data_dict["aspect_ratio"] + aug_size = augmentor_cfg["size"][aspect_ratio] + return aug_size diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/image/normalize.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/normalize.py new file mode 100644 index 00000000..0c50d3a5 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/normalize.py @@ -0,0 +1,48 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import torch +import torchvision.transforms.functional as transforms_F +from nemo.collections.physicalai.tokenizer.data.augmentors.augmentor import Augmentor + + +class Normalize(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs data normalization. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are center cropped. + """ + assert self.args is not None, "Please specify args" + + mean = self.args["mean"] + std = self.args["std"] + + for key in self.input_keys: + if isinstance(data_dict[key], torch.Tensor): + data_dict[key] = data_dict[key].to(dtype=torch.get_default_dtype()).div(255) + else: + data_dict[key] = transforms_F.to_tensor(data_dict[key]) # division by 255 is applied in to_tensor() + + data_dict[key] = transforms_F.normalize(tensor=data_dict[key], mean=mean, std=std) + return data_dict diff --git a/nemo_vfm/physicalai/tokenizer/data/augmentors/image/resize.py b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/resize.py new file mode 100644 index 00000000..964ed370 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/augmentors/image/resize.py @@ -0,0 +1,186 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from typing import Optional + +import omegaconf +import torchvision.transforms.functional as transforms_F +from nemo.collections.physicalai.tokenizer.data.augmentors.augmentor import Augmentor +from nemo.collections.physicalai.tokenizer.data.augmentors.image.misc import ( + obtain_augmentation_size, + obtain_image_size, +) + + +class ResizeSmallestSide(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs resizing to smaller side + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are resized + """ + + if self.output_keys is None: + self.output_keys = self.input_keys + assert self.args is not None, "Please specify args in augmentations" + + for inp_key, out_key in zip(self.input_keys, self.output_keys): + out_size = obtain_augmentation_size(data_dict, self.args) + assert isinstance(out_size, int), "Arg size in resize should be an integer" + data_dict[out_key] = transforms_F.resize( + data_dict[inp_key], + size=out_size, # type: ignore + interpolation=getattr(self.args, "interpolation", transforms_F.InterpolationMode.BICUBIC), + antialias=True, + ) + if out_key != inp_key: + del data_dict[inp_key] + return data_dict + + +class ResizeLargestSide(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs resizing to larger side + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are resized + """ + + if self.output_keys is None: + self.output_keys = self.input_keys + assert self.args is not None, "Please specify args in augmentations" + + for inp_key, out_key in zip(self.input_keys, self.output_keys): + out_size = obtain_augmentation_size(data_dict, self.args) + assert isinstance(out_size, int), "Arg size in resize should be an integer" + orig_w, orig_h = obtain_image_size(data_dict, self.input_keys) + + scaling_ratio = min(out_size / orig_w, out_size / orig_h) + target_size = [int(scaling_ratio * orig_h), int(scaling_ratio * orig_w)] + + data_dict[out_key] = transforms_F.resize( + data_dict[inp_key], + size=target_size, + interpolation=getattr(self.args, "interpolation", transforms_F.InterpolationMode.BICUBIC), + antialias=True, + ) + if out_key != inp_key: + del data_dict[inp_key] + return data_dict + + +class ResizeSmallestSideAspectPreserving(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs aspect-ratio preserving resizing. + Image is resized to the dimension which has the smaller ratio of (size / target_size). + First we compute (w_img / w_target) and (h_img / h_target) and resize the image + to the dimension that has the smaller of these ratios. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are resized + """ + + if self.output_keys is None: + self.output_keys = self.input_keys + assert self.args is not None, "Please specify args in augmentations" + + img_size = obtain_augmentation_size(data_dict, self.args) + assert isinstance(img_size, (tuple, omegaconf.listconfig.ListConfig)), ( + f"Arg size in resize should be a tuple, get {type(img_size)}, {img_size}" + ) + img_w, img_h = img_size + + orig_w, orig_h = obtain_image_size(data_dict, self.input_keys) + scaling_ratio = max((img_w / orig_w), (img_h / orig_h)) + target_size = (int(scaling_ratio * orig_h + 0.5), int(scaling_ratio * orig_w + 0.5)) + + assert target_size[0] >= img_h and target_size[1] >= img_w, ( + f"Resize error. orig {(orig_w, orig_h)} desire {img_size} compute {target_size}" + ) + + for inp_key, out_key in zip(self.input_keys, self.output_keys): + data_dict[out_key] = transforms_F.resize( + data_dict[inp_key], + size=target_size, # type: ignore + interpolation=getattr(self.args, "interpolation", transforms_F.InterpolationMode.BICUBIC), + antialias=True, + ) + + if out_key != inp_key: + del data_dict[inp_key] + return data_dict + + +class ResizeLargestSideAspectPreserving(Augmentor): + def __init__(self, input_keys: list, output_keys: Optional[list] = None, args: Optional[dict] = None) -> None: + super().__init__(input_keys, output_keys, args) + + def __call__(self, data_dict: dict) -> dict: + r"""Performs aspect-ratio preserving resizing. + Image is resized to the dimension which has the larger ratio of (size / target_size). + First we compute (w_img / w_target) and (h_img / h_target) and resize the image + to the dimension that has the larger of these ratios. + + Args: + data_dict (dict): Input data dict + Returns: + data_dict (dict): Output dict where images are resized + """ + + if self.output_keys is None: + self.output_keys = self.input_keys + assert self.args is not None, "Please specify args in augmentations" + + img_size = obtain_augmentation_size(data_dict, self.args) + assert isinstance(img_size, (tuple, omegaconf.listconfig.ListConfig)), ( + f"Arg size in resize should be a tuple, get {type(img_size)}, {img_size}" + ) + img_w, img_h = img_size + + orig_w, orig_h = obtain_image_size(data_dict, self.input_keys) + scaling_ratio = min((img_w / orig_w), (img_h / orig_h)) + target_size = (int(scaling_ratio * orig_h + 0.5), int(scaling_ratio * orig_w + 0.5)) + + assert target_size[0] <= img_h and target_size[1] <= img_w, ( + f"Resize error. orig {(orig_w, orig_h)} desire {img_size} compute {target_size}" + ) + + for inp_key, out_key in zip(self.input_keys, self.output_keys): + data_dict[out_key] = transforms_F.resize( + data_dict[inp_key], + size=target_size, # type: ignore + interpolation=getattr(self.args, "interpolation", transforms_F.InterpolationMode.BICUBIC), + antialias=True, + ) + + if out_key != inp_key: + del data_dict[inp_key] + return data_dict diff --git a/nemo_vfm/physicalai/tokenizer/data/dataloader.py b/nemo_vfm/physicalai/tokenizer/data/dataloader.py new file mode 100644 index 00000000..c9b85da0 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/dataloader.py @@ -0,0 +1,42 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import nemo_run as run +from nemo.collections.diffusion.data.diffusion_energon_datamodule import DiffusionDataModule +from nemo.collections.physicalai.tokenizer.train_tokenizer import ImageTaskEncoder +from tqdm import tqdm + + +@run.cli.entrypoint +def main(path, num_workers=16): + datamodule = DiffusionDataModule( + path=path, + task_encoder=ImageTaskEncoder(), + global_batch_size=2, + micro_batch_size=1, + num_workers=num_workers, + ) + + train = datamodule.train_dataloader() + for i in tqdm(iter(train)): + pass + from IPython import embed + + embed() + + +if __name__ == "__main__": + run.cli.main(main) diff --git a/nemo_vfm/physicalai/tokenizer/data/utils.py b/nemo_vfm/physicalai/tokenizer/data/utils.py new file mode 100644 index 00000000..cbc6b6d7 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/data/utils.py @@ -0,0 +1,127 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""Utilities for datasets creation.""" + +IMAGE_KEY = "images" +VIDEO_KEY = "video" +RECON_KEY = "reconstructions" +INPUT_KEY = "INPUT" +MASK_KEY = "loss_mask" + +_SPATIAL_ALIGN = 16 + + +def get_crop_size_info(crop_sz: int = 128): + aspect_ratios = [(1, 1), (4, 3), (3, 4), (16, 9), (9, 16)] + crop_sizes = dict() + for aspect_ratio in aspect_ratios: + if aspect_ratio[0] > aspect_ratio[1]: + crop_h = crop_sz // _SPATIAL_ALIGN * _SPATIAL_ALIGN + crop_w = int(crop_h * aspect_ratio[0] / aspect_ratio[1] + 0.5) + crop_w = crop_w // _SPATIAL_ALIGN * _SPATIAL_ALIGN + else: + crop_w = crop_sz // _SPATIAL_ALIGN * _SPATIAL_ALIGN + crop_h = int(crop_w * aspect_ratio[1] / aspect_ratio[0] + 0.5) + crop_h = crop_h // _SPATIAL_ALIGN * _SPATIAL_ALIGN + key = f"{aspect_ratio[0]},{aspect_ratio[1]}" + crop_sizes.update({key: (crop_w, crop_h)}) + return crop_sizes + + +IMAGE_RES_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": { + "1,1": (1080, 1080), + "4,3": (1440, 1080), + "3,4": (1080, 1440), + "16,9": (1920, 1080), + "9,16": (1080, 1920), + }, + "1024": {"1,1": (1024, 1024), "4,3": (1280, 1024), "3,4": (1024, 1280), "16,9": (1280, 768), "9,16": (768, 1280)}, + "720": {"1,1": (720, 720), "4,3": (960, 720), "3,4": (720, 960), "16,9": (1280, 720), "9,16": (720, 1280)}, + "480": {"1,1": (480, 480), "4,3": (640, 480), "3,4": (480, 640), "16,9": (854, 480), "9,16": (480, 854)}, + "320": {"1,1": (304, 304), "4,3": (400, 304), "3,4": (304, 400), "16,9": (544, 304), "9,16": (304, 544)}, + "512": {"1,1": (512, 512), "4,3": (672, 512), "3,4": (512, 672), "16,9": (896, 512), "9,16": (512, 896)}, + "256": {"1,1": (256, 256), "4,3": (320, 256), "3,4": (256, 320), "16,9": (320, 192), "9,16": (192, 320)}, + "128": { # Note that we set res lower than 256 to the same resolution as 256 + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (448, 256), + "9,16": (256, 448), + }, +} + +IMAGE_VAL_CROP_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": {"1,1": (720, 720), "4,3": (960, 720), "3,4": (720, 960), "16,9": (1280, 720), "9,16": (720, 1280)}, + "1024": {"1,1": (1024, 1024), "4,3": (1280, 1024), "3,4": (1024, 1280), "16,9": (1280, 768), "9,16": (768, 1280)}, + "720": {"1,1": (720, 720), "4,3": (960, 720), "3,4": (720, 960), "16,9": (1280, 720), "9,16": (720, 1280)}, + "480": {"1,1": (480, 480), "4,3": (640, 480), "3,4": (480, 640), "16,9": (848, 480), "9,16": (480, 848)}, + "320": {"1,1": (304, 304), "4,3": (400, 304), "3,4": (304, 400), "16,9": (544, 304), "9,16": (304, 544)}, + "512": {"1,1": (512, 512), "4,3": (672, 512), "3,4": (512, 672), "16,9": (896, 512), "9,16": (512, 896)}, + "256": {"1,1": (256, 256), "4,3": (320, 256), "3,4": (256, 320), "16,9": (320, 192), "9,16": (192, 320)}, + "128": { # Note that we set res lower than 256 to the same resolution as 256 + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (448, 256), + "9,16": (256, 448), + }, +} + +VIDEO_RES_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": { # 1080p doesn't have 1:1 + "4,3": (1440, 1072), + "3,4": (1072, 1440), + "16,9": (1920, 1072), + "9,16": (1072, 1920), + }, + "720": {"1,1": (720, 720), "4,3": (960, 720), "3,4": (720, 960), "16,9": (1280, 720), "9,16": (720, 1280)}, + "480": {"1,1": (480, 480), "4,3": (640, 480), "3,4": (480, 640), "16,9": (854, 480), "9,16": (480, 854)}, + "512": {"1,1": (512, 512), "4,3": (672, 512), "3,4": (512, 672), "16,9": (896, 512), "9,16": (512, 896)}, + "320": {"1,1": (320, 320), "4,3": (416, 320), "3,4": (320, 416), "16,9": (544, 320), "9,16": (320, 544)}, + "256": {"1,1": (256, 256), "4,3": (320, 256), "3,4": (256, 320), "16,9": (320, 192), "9,16": (192, 320)}, + "128": { # Note that we set res lower than 256 to the same resolution as 256 + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (448, 256), + "9,16": (256, 448), + }, +} + +VIDEO_VAL_CROP_SIZE_INFO: dict[str, tuple[int, int]] = { + "1080": { # 1080p doesn't have 1:1 + "4,3": (1424, 1072), + "3,4": (1072, 1424), + "16,9": (1904, 1072), + "9,16": (1072, 1904), + "16,10": (1715, 1072), + }, + "720": {"1,1": (704, 704), "4,3": (944, 704), "3,4": (704, 944), "16,9": (1264, 704), "9,16": (704, 1264)}, + "480": {"1,1": (464, 464), "4,3": (624, 464), "3,4": (464, 624), "16,9": (848, 464), "9,16": (464, 848)}, + "320": {"1,1": (320, 320), "4,3": (416, 320), "3,4": (320, 416), "16,9": (544, 320), "9,16": (320, 544)}, + "512": {"1,1": (512, 512), "4,3": (672, 512), "3,4": (512, 672), "16,9": (896, 512), "9,16": (512, 896)}, + "256": {"1,1": (256, 256), "4,3": (320, 256), "3,4": (256, 320), "16,9": (320, 192), "9,16": (192, 320)}, + "128": { # Note that we set res lower than 256 to the same resolution as 256 + "1,1": (256, 256), + "4,3": (320, 256), + "3,4": (256, 320), + "16,9": (320, 192), + "9,16": (192, 320), + "16,10": (410, 256), + }, +} diff --git a/nemo_vfm/physicalai/tokenizer/losses/__init__.py b/nemo_vfm/physicalai/tokenizer/losses/__init__.py new file mode 100644 index 00000000..b8178151 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/losses/__init__.py @@ -0,0 +1,51 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""The loss reduction modes.""" + +from enum import Enum + +import torch + + +def _mean(recon: torch.Tensor) -> torch.Tensor: + return torch.mean(recon) + + +def _sum_per_frame(recon: torch.Tensor) -> torch.Tensor: + batch_size = recon.shape[0] * recon.shape[2] if recon.ndim == 5 else recon.shape[0] + return torch.sum(recon) / batch_size + + +def _sum(recon: torch.Tensor) -> torch.Tensor: + return torch.sum(recon) / recon.shape[0] + + +class ReduceMode(Enum): + MEAN = "MEAN" + SUM_PER_FRAME = "SUM_PER_FRAME" + SUM = "SUM" + + @property + def function(self): + if self == ReduceMode.MEAN: + return _mean + elif self == ReduceMode.SUM_PER_FRAME: + return _sum_per_frame + elif self == ReduceMode.SUM: + return _sum + else: + raise ValueError("Invalid ReduceMode") diff --git a/nemo_vfm/physicalai/tokenizer/losses/config.py b/nemo_vfm/physicalai/tokenizer/losses/config.py new file mode 100644 index 00000000..6d14a20c --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/losses/config.py @@ -0,0 +1,74 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import attrs +from nemo.collections.physicalai.tokenizer.losses.loss import ReduceMode + + +@attrs.define(slots=False) +class ColorConfig: + # Color (RGB) basic loss and its weight schedule. + norm: str = "L1" + boundaries: list[int] = [0] + values: list[float] = [1.0] # TODO: confirm with fitsum + + +@attrs.define(slots=False) +class PerceptualConfig: + lpips_boundaries: list[int] = [500000] # TODO: confirm with fitsum + lpips_values: list[float] = [0.1, 0.073] + # Layer weights for linearly combining the multi-layer vgg-based losses. + layer_weights: list[float] = [1.0 / 2.6, 1.0 / 4.8, 1.0 / 3.7, 1.0 / 5.6, 10.0 / 1.5] + # Gram loss, whether to turn on, and what weights to use. + gram_loss: bool = True + gram_boundaries: list[int] = [500000] + gram_values: list[float] = [0.0, 0.062] + # Corr loss, whether to turn on, and what weights to use. + corr_loss: bool = False + corr_boundaries: list[int] = [0] + corr_values: list[float] = [0.0] + # In the example training memory usage dropped from 64.03 GiB to 60.54 GiB + # with checkpointing enabled for this loss for about 3.2% slowdown. + # With checkpointing this and PerceptualLoss memory usage dropped + # from 64.03 GiB to 52.94 GiB for about 18% slowdown + # more details in MR:949 + checkpoint_activations: bool = False + + +@attrs.define(slots=False) +class FlowConfig: + # Flow loss and its weight schedule. + boundaries: list[int] = [250000] # TODO: confirm with fitsum + values: list[float] = [0.0, 0.01] + scale: int = 2 + # Flow loss depends on RAFT, as such it requires a specific dtype. + dtype: str = "bfloat16" + # In the example training memory usage dropped from 28GB to 23GB + # with checkpointing enabled for this loss + # With checkpointing this and PerceptualLoss memory usage dropped + # from 64.03 GiB to 52.94 GiB for about 18% slowdown + # more details in MR:949 + checkpoint_activations: bool = False + + +@attrs.define(slots=False) +class VideoLoss: + # The combined loss function, and its reduction mode. + color: ColorConfig = ColorConfig() + perceptual: PerceptualConfig = PerceptualConfig() + flow: FlowConfig = FlowConfig() + reduce: str = ReduceMode.MEAN.value # model.config.loss.config.reduce={'MEAN', 'SUM', 'SUM_PER_FRAME'} + start_human_mask: int = 500000 # iteration when we enable human loss mask: TODO: diff --git a/nemo_vfm/physicalai/tokenizer/losses/loss.py b/nemo_vfm/physicalai/tokenizer/losses/loss.py new file mode 100644 index 00000000..e31c94a6 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/losses/loss.py @@ -0,0 +1,403 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +from enum import Enum + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +import torchvision.models.optical_flow as optical_flow +from einops import rearrange +from nemo.collections.physicalai.tokenizer.losses.lpips import LPIPS + + +IMAGE_KEY = "images" +VIDEO_KEY = "video" +RECON_KEY = "reconstructions" +LATENT_KEY = "latent" +INPUT_KEY = "INPUT" +MASK_KEY = "loss_mask" +RECON_CONSISTENCY_KEY = f"{RECON_KEY}_consistency" + + +def time2batch(x: torch.Tensor) -> tuple[torch.Tensor, int]: + batch_size = x.shape[0] + return rearrange(x, "b c t h w -> (b t) c h w"), batch_size + + +def batch2time(x: torch.Tensor, batch_size: int) -> torch.Tensor: + return rearrange(x, "(b t) c h w -> b c t h w", b=batch_size) + + +class WeightScheduler(torch.nn.Module): + def __init__(self, boundaries, values): + super().__init__() + self.boundaries = list(boundaries) + self.values = list(values) + + def forward(self, iteration): + for boundary, value in zip(self.boundaries, self.values): + if iteration < boundary: + return value + return self.values[-1] + + +class ColorLoss(torch.nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.schedule = WeightScheduler(boundaries=config.boundaries, values=config.values) + + def forward(self, inputs, output_batch, iteration) -> dict[str, torch.Tensor]: + reconstructions = output_batch[RECON_KEY] + weights = inputs[MASK_KEY] + recon = weights * torch.abs(inputs[INPUT_KEY].contiguous() - reconstructions.contiguous()) + color_weighted = self.schedule(iteration) * recon + if torch.isnan(color_weighted).any(): + raise ValueError("[COLOR] NaN detected in loss") + return dict(color=color_weighted) + + +class PerceptualLoss(LPIPS): + """Relevant changes that're internal to us: + + - Remove linear projection layers, simply use the raw pre-normalized features. + - Use pyramid-layer weights: [1.0 / 2.6, 1.0 / 4.8, 1.0 / 3.7, 1.0 / 5.6, 10.0 / 1.5]. + - Accepts pixel-wise masks and modulates the features before norm calculation. + - Implements gram-matrix and correlation losses. + """ + + def __init__(self, config): + super(PerceptualLoss, self).__init__(config.checkpoint_activations) + self.net = self.net.eval() + self.gram_loss = config.gram_loss + self.corr_loss = config.corr_loss + self.layer_weights = list(config.layer_weights) + self.lpips_schedule = WeightScheduler(config.lpips_boundaries, config.lpips_values) + self.gram_schedule = WeightScheduler(config.gram_boundaries, config.gram_values) + self.corr_schedule = WeightScheduler(config.corr_boundaries, config.corr_values) + self.checkpoint_activations = config.checkpoint_activations + + def _temporal_gram_matrix(self, x, batch_size=None): + x = batch2time(x, batch_size) + c, t, h, w = x.shape[-4], x.shape[-3], x.shape[-2], x.shape[-1] + reshaped_x = torch.reshape(x, [-1, c, t * h * w]) + return torch.matmul(reshaped_x, reshaped_x.transpose(1, 2)) / float(t * h * w) + + def _gram_matrix(self, x, batch_size=None): + if batch_size is not None and x.shape[0] != batch_size: + return self._temporal_gram_matrix(x, batch_size) + c, h, w = x.shape[-3], x.shape[-2], x.shape[-1] + reshaped_x = torch.reshape(x, [-1, c, h * w]) + return torch.matmul(reshaped_x, reshaped_x.transpose(1, 2)) / float(h * w) + + def forward(self, inputs, output_batch, iteration): + output_dict = dict() + reconstructions = output_batch[RECON_KEY] + weights = inputs[MASK_KEY] + input_images = inputs[INPUT_KEY] + + if input_images.ndim == 5: + input_images, batch_size = time2batch(input_images) + reconstructions, _ = time2batch(reconstructions) + weights, _ = time2batch(weights) + else: + batch_size = input_images.shape[0] + + in0_input, in1_input = (self.scaling_layer(input_images), self.scaling_layer(reconstructions)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + + _layer_weights = self.layer_weights + weights_map, res, diffs = {}, {}, {} + for kk in range(len(self.chns)): + weights_map[kk] = torch.nn.functional.interpolate(weights[:, :1, ...], outs0[kk].shape[-2:]) + diffs[kk] = weights_map[kk] * torch.abs(outs0[kk] - outs1[kk]) + res[kk] = _layer_weights[kk] * diffs[kk].mean([1, 2, 3], keepdim=True) + + val = res[0] + for ll in range(1, len(self.chns)): + val += res[ll] + # Scale by number of pixels to match pixel-wise losses. + val = val.expand(-1, input_images.shape[-3], input_images.shape[-2], input_images.shape[-1]) + if batch_size != input_images.shape[0]: + val = batch2time(val, batch_size) + if torch.isnan(val).any(): + raise ValueError("[LPIPS] NaN detected in loss") + output_dict["lpips"] = self.lpips_schedule(iteration) * val + + if self.gram_loss and self.gram_schedule(iteration) > 0.0: + num_chans = len(self.chns) + grams0 = [self._gram_matrix(weights_map[kk] * outs0[kk], batch_size) for kk in range(num_chans)] + grams1 = [self._gram_matrix(weights_map[kk] * outs1[kk], batch_size) for kk in range(num_chans)] + gram_diffs = [(grams0[kk] - grams1[kk]) ** 2 for kk in range(num_chans)] + grams_res = [_layer_weights[kk] * gram_diffs[kk].mean([1, 2], keepdim=True) for kk in range(num_chans)] + gram_val = grams_res[0] + for ll in range(1, len(self.chns)): + gram_val += grams_res[ll] + + # Scale by number of total pixels to match pixel-wise losses. + gram_val = gram_val.unsqueeze(1).expand( + -1, input_images.shape[-3], input_images.shape[-2], input_images.shape[-1] + ) + if batch_size != input_images.shape[0]: + gram_val = batch2time(gram_val, batch_size) + if torch.isnan(gram_val).any(): + raise ValueError("[GRAM] NaN detected in loss") + output_dict["gram"] = self.gram_schedule(iteration) * gram_val + return output_dict + + def torch_compile(self): + """ + This method invokes torch.compile() on this loss + """ + # cuda-graphs crash after 1k iterations + self.net = torch.compile(self.net, dynamic=False) + + +class FlowLoss(torch.nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.schedule = WeightScheduler(config.boundaries, config.values) + self.scale = config.scale + self.dtype = getattr(torch, config.dtype) + self.checkpoint_activations = config.checkpoint_activations + + current_device = torch.device(torch.cuda.current_device()) + + # In order to be able to run model in bf16 we need to change make_coords_grid() + # to allow it to return arbitrary type provided by us in argument + # the line from orginal implementation that caused results to be only fp32 is commented + # Additionally I've changed that function to run on GPU instead of CPU, which results in + # less graph breaks when torch.compile() is used + # This function is copied from + # https://github.com/pytorch/vision/blob/main/torchvision/models/optical_flow/_utils.py#L22 + # commit: b06ea39d5f0adbe949d08257837bda912339e415 + def make_coords_grid( + batch_size: int, h: int, w: int, device: torch.device = current_device, dtype: torch.dtype = self.dtype + ): + # Original: def make_coords_grid(batch_size: int, h: int, w: int, device: str = "cpu"): + device = torch.device(device) + coords = torch.meshgrid(torch.arange(h, device=device), torch.arange(w, device=device), indexing="ij") + coords = torch.stack(coords[::-1], dim=0).to(dtype) + # Original: coords = torch.stack(coords[::-1], dim=0).float() + return coords[None].repeat(batch_size, 1, 1, 1) + + # We also need to specify output dtype of torch.linspace() in index_pyramid() + # method of CorrBlock, otherwise it uses default fp32 dtype as output. + # Additionally I've changed that function to run on GPU instead of CPU, which results in + # less graph breaks when torch.compile() is used + # This function is copied from + # https://github.com/pytorch/vision/blob/main/torchvision/models/optical_flow/raft.py#L394 + # commit: b06ea39d5f0adbe949d08257837bda912339e415 + def index_pyramid( + self, centroids_coords, dtype: torch.dtype = self.dtype, device: torch.device = current_device + ): + # Original: def index_pyramid(self, centroids_coords): + """Return correlation features by indexing from the pyramid.""" + neighborhood_side_len = 2 * self.radius + 1 # see note in __init__ about out_channels + di = torch.linspace(-self.radius, self.radius, neighborhood_side_len, dtype=dtype, device=device) + dj = torch.linspace(-self.radius, self.radius, neighborhood_side_len, dtype=dtype, device=device) + # Original: di = torch.linspace(-self.radius, self.radius, neighborhood_side_len) + # Original: dj = torch.linspace(-self.radius, self.radius, neighborhood_side_len) + delta = torch.stack(torch.meshgrid(di, dj, indexing="ij"), dim=-1).to(centroids_coords.device) + delta = delta.view(1, neighborhood_side_len, neighborhood_side_len, 2) + + batch_size, _, h, w = centroids_coords.shape # _ = 2 + centroids_coords = centroids_coords.permute(0, 2, 3, 1).reshape(batch_size * h * w, 1, 1, 2) + + indexed_pyramid = [] + for corr_volume in self.corr_pyramid: + sampling_coords = centroids_coords + delta # end shape is (batch_size * h * w, side_len, side_len, 2) + indexed_corr_volume = optical_flow.raft.grid_sample( + corr_volume, sampling_coords, align_corners=True, mode="bilinear" + ).view(batch_size, h, w, -1) + indexed_pyramid.append(indexed_corr_volume) + centroids_coords = centroids_coords / 2 + + corr_features = torch.cat(indexed_pyramid, dim=-1).permute(0, 3, 1, 2).contiguous() + + expected_output_shape = (batch_size, self.out_channels, h, w) + if corr_features.shape != expected_output_shape: + raise ValueError( + f"Output shape of index pyramid is incorrect. Should be {expected_output_shape}, got {corr_features.shape}" + ) + + return corr_features + + optical_flow.raft.make_coords_grid = make_coords_grid + optical_flow.raft.CorrBlock.index_pyramid = index_pyramid + + flow_model = optical_flow.raft_large(pretrained=True, progress=False) + flow_model.requires_grad_(False) + flow_model.eval() + flow_model = flow_model.to(self.dtype) + + self.flow_model = flow_model + + def _run_model(self, input1: torch.Tensor, input2: torch.Tensor) -> torch.Tensor: + """Runs flow_model in the forward mode on explicit dtype=float32. + + Args: + input1: First video frames batch, layout (T, C, H, W), bfloat16. + input2: Next video frames batch, layout (T, C, H, W), bfloat16. + + Returns: + Forward optical flow, (T, 2, H, W), bfloat16. + """ + input_dtype = input1.dtype + flow_output = self.flow_model.to(self.dtype)(input1.to(self.dtype), input2.to(self.dtype))[-1] + return flow_output.to(input_dtype) + + def _run_model_fwd(self, input_video: torch.Tensor) -> torch.Tensor: + """Runs foward flow on a batch of videos, one batch at a time. + Args: + input_video: The input batch of videos, layout (B, T, C, H, W). + + Returns: + Forward optical flow, layout (B, 2, T-1, H, W). + """ + output_list = list() + for fwd_input_frames in input_video: + fwd_input_frames = fwd_input_frames.transpose(1, 0) + fwd_flow_output = self._run_model(fwd_input_frames[:-1], fwd_input_frames[1:]) + output_list.append(fwd_flow_output.transpose(1, 0)) + return torch.stack(output_list, dim=0) + + def _bidirectional_flow(self, input_video: torch.Tensor) -> torch.Tensor: + """The bidirectional optical flow on a batch of videos. + + The forward and backward flows are averaged to get the bidirectional flow. + To reduce memory pressure, the input video is scaled down by a factor of `self.scale`, + and rescaled back to match other pixel-wise losses. + + Args: + input_video: The input batch of videos, layout (B, T, C, H, W). + + Returns: + Biderectinoal flow, layout (B, 2, T-1, H, W). + """ + # scale down the input video to reduce memory pressure. + t, h, w = input_video.shape[-3:] + input_video_scaled = F.interpolate(input_video, (t, h // self.scale, w // self.scale), mode="trilinear") + + # forward flow. + if self.checkpoint_activations: + fwd_flow_output = checkpoint.checkpoint(self._run_model_fwd, input_video_scaled, use_reentrant=False) + else: + fwd_flow_output = self._run_model_fwd(input_video_scaled) + + # backward flow. + input_video_scaled = input_video_scaled.flip([2]) + if self.checkpoint_activations: + bwd_flow_output = checkpoint.checkpoint(self._run_model_fwd, input_video_scaled, use_reentrant=False) + else: + bwd_flow_output = self._run_model_fwd(input_video_scaled) + bwd_flow_output = bwd_flow_output.flip([2]) + + # bidirectional flow, concat fwd and bwd along temporal axis. + flow_input = torch.cat([fwd_flow_output, bwd_flow_output], dim=2) + return self.scale * F.interpolate(flow_input, (2 * (t - 1), h, w), mode="trilinear") + + def forward( + self, inputs: dict[str, torch.Tensor], output_batch: dict[str, torch.Tensor], iteration: int + ) -> dict[str, torch.Tensor]: + input_images = inputs[INPUT_KEY] + if input_images.ndim == 4 or input_images.shape[2] == 1: + return dict() + if self.schedule(iteration) == 0.0: + return dict() + + # Biderectional flow (B, 2, 2*(T-1), H, W) + flow_input = self._bidirectional_flow(input_images) + flow_recon = self._bidirectional_flow(output_batch[RECON_KEY]) + + # L1 loss on the flow. (B, 1, 2*(T-1), H, W) + flow_loss = torch.abs(flow_input - flow_recon).mean(dim=1, keepdim=True) + + flow_loss_weighted = self.schedule(iteration) * flow_loss + if torch.isnan(flow_loss_weighted).any(): + raise ValueError("[FLOW] NaN detected in loss") + return dict(flow=flow_loss_weighted) + + def torch_compile(self): + """ + This method invokes torch.compile() on this loss + """ + self.flow_model = torch.compile(self.flow_model, dynamic=False) + + +###################################### +_VALID_LOSS_NAMES = ["color", "perceptual", "flow", "kl", "video_consistency"] + + +def _mean(recon: torch.Tensor) -> torch.Tensor: + return torch.mean(recon) + + +def _sum_per_frame(recon: torch.Tensor) -> torch.Tensor: + batch_size = recon.shape[0] * recon.shape[2] if recon.ndim == 5 else recon.shape[0] + return torch.sum(recon) / batch_size + + +def _sum(recon: torch.Tensor) -> torch.Tensor: + return torch.sum(recon) / recon.shape[0] + + +class ReduceMode(Enum): + MEAN = "MEAN" + SUM_PER_FRAME = "SUM_PER_FRAME" + SUM = "SUM" + + @property + def function(self): + if self == ReduceMode.MEAN: + return _mean + elif self == ReduceMode.SUM_PER_FRAME: + return _sum_per_frame + elif self == ReduceMode.SUM: + return _sum + else: + raise ValueError("Invalid ReduceMode") + + +class TokenizerLoss(nn.Module): + def __init__(self, config) -> None: + super().__init__() + self.config = config + _reduce = ReduceMode(config.reduce.upper()) if hasattr(config, "reduce") else None + self.reduce = _reduce.function + + # manually create these losses + self.loss_modules = nn.ModuleDict() + self.loss_modules["color"] = ColorLoss(config=config.color) + self.loss_modules["perceptual"] = PerceptualLoss(config=config.perceptual) + self.loss_modules["flow"] = FlowLoss(config=config.flow) + + def forward(self, inputs, output_batch, iteration) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + loss = dict() + total_loss = 0.0 + + # human loss_mask is only enabled after iter=start_human_mask + if iteration < self.config.start_human_mask: + inputs[MASK_KEY] = torch.ones_like(inputs[INPUT_KEY]) + + # Calculates reconstruction losses (`total_loss`). + for key, module in self.loss_modules.items(): + curr_loss = module(inputs, output_batch, iteration) + loss.update({k: torch.mean(v) for k, v in curr_loss.items()}) + total_loss += sum([self.reduce(v) if (v.dim() > 0) else v for v in curr_loss.values()]) + + return dict(loss=loss), total_loss diff --git a/nemo_vfm/physicalai/tokenizer/losses/lpips.py b/nemo_vfm/physicalai/tokenizer/losses/lpips.py new file mode 100644 index 00000000..90d5bf09 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/losses/lpips.py @@ -0,0 +1,212 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""LPIPS loss. + +Adapted from: github.com/CompVis/stable-diffusion/ldm/modules/losses/contperceptual.py. +""" + +import hashlib +import os +from collections import namedtuple +from typing import Optional + +import requests +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.utils.checkpoint as checkpoint +from loguru import logger as logging +from torchvision import models +from tqdm import tqdm + + +def get_rank(group: Optional[dist.ProcessGroup] = None) -> int: + """Get the rank (GPU device) of the worker. + + Returns: + rank (int): The rank of the worker. + """ + rank = 0 + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank(group) + return rank + + +def is_rank0() -> bool: + """Check if current process is the master GPU. + + Returns: + (bool): True if this function is called from the master GPU, else False. + """ + return get_rank() == 0 + + +_TORCH_HOME = os.getenv("TORCH_HOME", "~/.cache/my_model") +# TODO(freda): Update the download link to a PBSS location, safer. +_URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"} +_CKPT_MAP = {"vgg_lpips": "vgg.pth"} +_MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"} + + +def _download(url, local_path, chunk_size=1024): + os.makedirs(os.path.split(local_path)[0], exist_ok=True) + with requests.get(url, stream=True) as r: + total_size = int(r.headers.get("content-length", 0)) + with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: + with open(local_path, "wb") as f: + for data in r.iter_content(chunk_size=chunk_size): + if data: + f.write(data) + pbar.update(chunk_size) + + +def _md5_hash(path): + with open(path, "rb") as f: + content = f.read() + return hashlib.md5(content).hexdigest() + + +def _get_ckpt_path(name, root, check=False): + assert name in _URL_MAP + path = os.path.join(root, _CKPT_MAP[name]) + if not os.path.exists(path) or (check and not _md5_hash(path) == _MD5_MAP[name]): + logging.info("Downloading {} model from {} to {}".format(name, _URL_MAP[name], path)) + _download(_URL_MAP[name], path) + md5 = _md5_hash(path) + assert md5 == _MD5_MAP[name], md5 + return path + + +class LPIPS(nn.Module): + def __init__(self, checkpoint_activations: bool = False): + super().__init__() + self.scaling_layer = ScalingLayer() + self.chns = [64, 128, 256, 512, 512] # vg16 features + self.net = vgg16(pretrained=True, requires_grad=False, checkpoint_activations=checkpoint_activations) + + if dist.is_initialized() and not is_rank0(): + dist.barrier() + self.load_from_pretrained() + if dist.is_initialized() and is_rank0(): + dist.barrier() + + for param in self.parameters(): + param.requires_grad = False + + def load_from_pretrained(self, name="vgg_lpips"): + ckpt = _get_ckpt_path(name, f"{_TORCH_HOME}/hub/checkpoints") + self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + logging.info("Loaded pretrained LPIPS loss from {}".format(ckpt)) + + @classmethod + def from_pretrained(cls, name="vgg_lpips"): + if name != "vgg_lpips": + raise NotImplementedError + model = cls() + ckpt = _get_ckpt_path(name) + model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + return model + + def forward(self, input, target): + in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + feats0, feats1, diffs = {}, {}, {} + for kk in range(len(self.chns)): + feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk]) + diffs[kk] = (feats0[kk] - feats1[kk]) ** 2 + + res = [diffs[kk].mean([1, 2, 3], keepdim=True) for kk in range(len(self.chns))] + val = res[0] + for i in range(1, len(self.chns)): + val += res[i] + return val + + +class ScalingLayer(nn.Module): + def __init__(self): + super(ScalingLayer, self).__init__() + self.register_buffer("shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None], persistent=False) + self.register_buffer("scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None], persistent=False) + + def forward(self, inp): + return (inp - self.shift) / self.scale + + +def normalize_tensor(x, eps=1e-10): + norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True)) + return x / (norm_factor + eps) + + +class vgg16(torch.nn.Module): + def __init__(self, requires_grad=False, pretrained=True, checkpoint_activations: bool = False): + super(vgg16, self).__init__() + vgg_pretrained_features = models.vgg16(pretrained=pretrained).features + self.checkpoint_activations = checkpoint_activations + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + self.N_slices = 5 + for x in range(4): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(4, 9): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(9, 16): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(16, 23): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(23, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + if self.checkpoint_activations: + h = checkpoint.checkpoint(self.slice1, X, use_reentrant=False) + else: + h = self.slice1(X) + h_relu1_2 = h + + if self.checkpoint_activations: + h = checkpoint.checkpoint(self.slice2, h, use_reentrant=False) + else: + h = self.slice2(h) + h_relu2_2 = h + + if self.checkpoint_activations: + h = checkpoint.checkpoint(self.slice3, h, use_reentrant=False) + else: + h = self.slice3(h) + h_relu3_3 = h + + if self.checkpoint_activations: + h = checkpoint.checkpoint(self.slice4, h, use_reentrant=False) + else: + h = self.slice4(h) + h_relu4_3 = h + + if self.checkpoint_activations: + h = checkpoint.checkpoint(self.slice5, h, use_reentrant=False) + else: + h = self.slice5(h) + h_relu5_3 = h + + vgg_outputs = namedtuple("VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"]) + out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3) + return out diff --git a/nemo_vfm/physicalai/tokenizer/losses/test.py b/nemo_vfm/physicalai/tokenizer/losses/test.py new file mode 100644 index 00000000..15946c01 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/losses/test.py @@ -0,0 +1,59 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from config import ColorConfig, FlowConfig, PerceptualConfig, VideoLoss +from loss import ColorLoss, FlowLoss, PerceptualLoss, TokenizerLoss + + +VIDEO_KEY = "video" +RECON_KEY = "reconstructions" +LATENT_KEY = "latent" +INPUT_KEY = "INPUT" +MASK_KEY = "loss_mask" +RECON_CONSISTENCY_KEY = f"{RECON_KEY}_consistency" + +# Setup input +device = torch.device("cuda") # Move to GPU + +input_t = torch.randn([2, 3, 10, 256, 256], dtype=torch.bfloat16, device=device) +mask_t = torch.ones_like(input_t, requires_grad=False, dtype=torch.bfloat16, device=device) +inputs = {INPUT_KEY: input_t, MASK_KEY: mask_t} + +reconstructions = torch.randn([2, 3, 10, 256, 256], dtype=torch.bfloat16, device=device) +output_batch = {RECON_KEY: reconstructions} + +# Create loss (assuming these loss functions support bf16) +colorLoss = ColorLoss(config=ColorConfig()).to(device).to(torch.bfloat16) +perceptualLoss = PerceptualLoss(config=PerceptualConfig()).to(device).to(torch.bfloat16) +flowLoss = FlowLoss(config=FlowConfig()).to(device).to(torch.bfloat16) +videoLoss = TokenizerLoss(config=VideoLoss()).to(device).to(torch.bfloat16) + +print("#" * 20) + +# Test color loss +cLoss = colorLoss(inputs, output_batch, 0) +print("color loss shape:", cLoss["color"].shape) + +# Test LPIPS loss +pLoss = perceptualLoss(inputs, output_batch, 0) +print("LPIPS loss:", pLoss.keys(), pLoss["lpips"].shape) + +# Test flow loss +fLoss = flowLoss(inputs, output_batch, 250001) +print("Flow loss:", fLoss.keys(), fLoss["flow"].shape) + +# Test video loss +vLoss, total_vLoss = videoLoss(inputs, output_batch, 250001) +print("Video loss:", vLoss.keys(), vLoss, total_vLoss) diff --git a/nemo_vfm/physicalai/tokenizer/test_tokenizer_model.py b/nemo_vfm/physicalai/tokenizer/test_tokenizer_model.py new file mode 100644 index 00000000..68ea5ead --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/test_tokenizer_model.py @@ -0,0 +1,58 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +import torch.distributed +from tokenizer_model import TokenizerModel + + +VIDEO_KEY = "video" +RECON_KEY = "reconstructions" +LATENT_KEY = "latent" +INPUT_KEY = "INPUT" +MASK_KEY = "loss_mask" +RECON_CONSISTENCY_KEY = f"{RECON_KEY}_consistency" + +tm = TokenizerModel( + jit_ckpt_pth=os.path.join( + os.environ["HF_HOME"], + "hub/models--nvidia--Cosmos-1.0-Tokenizer-CV8x8x8/snapshots/01f87fd67cebc32f1a2fd9e99d4e9614a6b3743b", + ) +) + +device = torch.device("cuda") # Move to GPU +input_t = torch.randn([2, 3, 33, 256, 256], dtype=torch.bfloat16, device=device) +mask_t = torch.ones_like(input_t, requires_grad=False, dtype=torch.bfloat16, device=device) +inputs = {VIDEO_KEY: input_t, MASK_KEY: mask_t} + +reconstructions = torch.randn([2, 3, 33, 256, 256], dtype=torch.bfloat16, device=device) +output_batch = {RECON_KEY: reconstructions} + +output = tm._training_step(inputs, 20000000) + +import wandb + + +wandb.init() +import torch +import torch.distributed as dist + + +torch.distributed.init_process_group(backend="nccl") +rank = dist.get_rank() +torch.cuda.set_device(rank) # Assign each process to a GPU + +tm.validation_step(inputs, 20000000) diff --git a/nemo_vfm/physicalai/tokenizer/tokenizer_model.py b/nemo_vfm/physicalai/tokenizer/tokenizer_model.py new file mode 100644 index 00000000..0135f9df --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/tokenizer_model.py @@ -0,0 +1,172 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +"""Implements the forward op for training, validation, and inference.""" + +import os +from typing import Optional + +import lightning.pytorch as L +import torch +import wandb +from cosmos1.models.tokenizer.networks.configs import continuous_video, discrete_video +from cosmos1.models.tokenizer.networks.continuous_video import CausalContinuousVideoTokenizer +from cosmos1.models.tokenizer.networks.discrete_video import CausalDiscreteVideoTokenizer +from einops import rearrange +from nemo.collections.llm import fn +from nemo.collections.physicalai.tokenizer.losses.config import VideoLoss +from nemo.collections.physicalai.tokenizer.losses.loss import TokenizerLoss +from nemo.lightning import io +from nemo.lightning.pytorch.optim import OptimizerModule + + +IMAGE_KEY = "images" +INPUT_KEY = "INPUT" +MASK_KEY = "loss_mask" +RECON_KEY = "reconstructions" +VIDEO_KEY = "video" + +RECON_CONSISTENCY_KEY = f"{RECON_KEY}_consistency" +VIDEO_CONSISTENCY_LOSS = "video_consistency" + +PREDICTION = "prediction" +EMA_PREDICTION = "ema_prediction" + + +class TokenizerModel(L.LightningModule, io.IOMixin, fn.FNMixin): + def __init__( + self, + jit_ckpt_pth=None, + model="Cosmos-1.0-Tokenizer-CV8x8x8", + precision=torch.bfloat16, + device=torch.device("cuda"), + optim: Optional[OptimizerModule] = None, + ) -> None: + super().__init__() + self.save_hyperparameters() + + autoencoder_path = os.path.join(jit_ckpt_pth, "autoencoder.jit") + if not os.path.exists(autoencoder_path): + raise FileNotFoundError(f"autoencoder.jit not found at {autoencoder_path}") + + weight = torch.jit.load(autoencoder_path) + if model == "Cosmos-1.0-Tokenizer-CV8x8x8": + self.network = CausalContinuousVideoTokenizer(**continuous_video) + elif model == "Cosmos-1.0-Tokenizer-DV8x16x16": + self.network = CausalDiscreteVideoTokenizer(**discrete_video) + else: + raise NotImplementedError( + "We only support Cosmos-1.0-Tokenizer-CV8x8x8 and Cosmos-1.0-Tokenizer-DV8x16x16" + ) + self.network.load_state_dict(weight.state_dict(), strict=False) + self.network = self.network.to(device=device, dtype=precision) + self.network.training = True + + self.loss = TokenizerLoss(config=VideoLoss()) + self.loss = self.loss.to(device=device, dtype=precision) + + self._precision = precision + self._device = device + + self.image_key = IMAGE_KEY + self.video_key = VIDEO_KEY + + def get_input_key(self, data_batch: dict[str, torch.Tensor]) -> str: + if self.video_key in data_batch: + return self.video_key + else: + raise ValueError("Input key not found in data_batch.") + + def on_train_start(self, memory_format: torch.memory_format = torch.preserve_format) -> None: + self.network = self.network.to(device=self._device, dtype=self._precision, memory_format=memory_format) + self.loss = self.loss.to(device=self._device, dtype=self._precision, memory_format=memory_format) + + def _network_forward(self, data_batch: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + # Do the forward pass. + tensor_batch = data_batch[self.get_input_key(data_batch)] + output_batch = self.network(tensor_batch) + output_batch = output_batch if self.network.training else output_batch._asdict() + + return output_batch + + def _training_step( + self, + data_batch: dict[str, torch.Tensor], + iteration: int, + ) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + _input_key = self.get_input_key(data_batch) + output_dict = self._network_forward(data_batch) + input_images, recon_images = data_batch[_input_key], output_dict[RECON_KEY] + + # pass loss_mask to loss computation + inputs = {INPUT_KEY: input_images, MASK_KEY: data_batch.get("loss_mask", torch.ones_like(input_images))} + loss_dict, loss_value = self.loss(inputs, output_dict, iteration) + for k, v in loss_dict["loss"].items(): + self.log(k, v) + self.log("loss", loss_value, prog_bar=True) + self.log("global_step", self.global_step) + + return dict({PREDICTION: recon_images, **loss_dict}), loss_value + + def training_step( + self, + data_batch: dict[str, torch.Tensor], + iteration: int, + ) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + _, loss_value = self._training_step(data_batch, iteration) + return loss_value + + # TODO: add validation step + @torch.no_grad() + def validation_step( + self, + data_batch: dict[str, torch.Tensor], + iteration: int, + ema_model: bool = False, + ) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + _input_key = self.get_input_key(data_batch) + output_dict = self._network_forward(data_batch) + input_images, recon_images = data_batch[_input_key], output_dict[RECON_KEY] + + # pass loss_mask to loss computation + inputs = {INPUT_KEY: input_images, MASK_KEY: data_batch.get("loss_mask", torch.ones_like(input_images))} + + loss_dict, loss_value = self.loss(inputs, output_dict, iteration) + + if wandb.run is not None: + visualization = torch.cat( + [ + rearrange(input_images[0], "c t h w -> t c h w").cpu(), + rearrange(recon_images[0], "c t h w -> t c h w").cpu(), + ], + axis=-1, + ) + visualization = ((visualization + 0.5).clamp(0, 1).cpu().float().numpy() * 255).astype("uint8") + + wandb.log( + { + "Original (left), Reconstruction (right)": [ + wandb.Video(visualization, fps=24), + ] + }, + ) + + return loss_value + + @torch.inference_mode() + def forward(self, data_batch: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + output_dict = self._network_forward(data_batch) + return dict({PREDICTION: output_dict[RECON_KEY]}) diff --git a/nemo_vfm/physicalai/tokenizer/train_tokenizer.py b/nemo_vfm/physicalai/tokenizer/train_tokenizer.py new file mode 100644 index 00000000..29d44043 --- /dev/null +++ b/nemo_vfm/physicalai/tokenizer/train_tokenizer.py @@ -0,0 +1,238 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import os + +import nemo_run as run +import pytorch_lightning as pl +import torch.distributed +import torch.utils.checkpoint +from einops import rearrange +from megatron.energon import DefaultTaskEncoder +from nemo import lightning as nl +from nemo.collections import llm +from nemo.collections.diffusion.data.diffusion_energon_datamodule import DiffusionDataModule +from nemo.collections.physicalai.tokenizer.augmentors import TemporalRandomCrop +from nemo.collections.physicalai.tokenizer.data.augmentors.image.cropping import RandomCrop +from nemo.collections.physicalai.tokenizer.data.augmentors.image.normalize import Normalize +from nemo.collections.physicalai.tokenizer.data.utils import get_crop_size_info +from nemo.collections.physicalai.tokenizer.tokenizer_model import MASK_KEY, VIDEO_KEY, TokenizerModel +from nemo.lightning.io.mixin import IOMixin +from nemo.lightning.pytorch.callbacks import ModelCheckpoint, PreemptionCallback +from nemo.lightning.pytorch.optim.pytorch import PytorchOptimizerModule +from nemo.lightning.pytorch.plugins import MegatronDataSampler +from nemo.utils.exp_manager import TimingCallback +from pytorch_lightning.loggers import WandbLogger +from torch.utils.data import DataLoader + + +class FakeDataset(torch.utils.data.Dataset): + def __init__(self): + super().__init__() + + def __len__(self): + return 100000000 + + def __getitem__(self, idx): + input_t = torch.randn([2, 3, 33, 256, 256], dtype=torch.bfloat16, device="cuda") + mask_t = torch.ones_like(input_t, requires_grad=False, dtype=torch.bfloat16, device="cuda") + return {VIDEO_KEY: input_t, MASK_KEY: mask_t} + + def _collate_fn(self, batch): + """ + A default implementation of a collation function. + Users should override this method to define custom data loaders. + """ + return torch.utils.data.dataloader.default_collate(batch) + + def collate_fn(self, batch): + """Method that user pass as functor to DataLoader. + + The method optionally performs neural type checking and add types to the outputs. + + Please note, subclasses of Dataset should not implement `input_types`. + + # Usage: + dataloader = torch.utils.data.DataLoader( + ...., + collate_fn=dataset.collate_fn, + .... + ) + + Returns + ------- + Collated batch, with or without types. + """ + return self._collate_fn(batch) + + +class FakeDataModule(pl.LightningDataModule): + def __init__( + self, + seq_length: int = 2048, + micro_batch_size: int = 1, + global_batch_size: int = 8, + num_workers: int = 1, + pin_memory: bool = True, + use_train_split_for_val: bool = False, + task_encoder=None, + ) -> None: + super().__init__() + self.seq_length = seq_length + self.micro_batch_size = micro_batch_size + self.global_batch_size = global_batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + + self.data_sampler = MegatronDataSampler( + seq_len=self.seq_length, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + ) + + def setup(self, stage: str = "") -> None: + self._train_ds = FakeDataset() + + def train_dataloader(self): + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def val_dataloader(self): + if not hasattr(self, "_train_ds"): + self.setup() + return self._create_dataloader(self._train_ds) + + def _create_dataloader(self, dataset, **kwargs): + return DataLoader( + dataset, + num_workers=self.num_workers, + pin_memory=True, + persistent_workers=True, + collate_fn=dataset.collate_fn, + **kwargs, + ) + + +class ImageTaskEncoder(DefaultTaskEncoder, IOMixin): + """Image task encoder that crops and normalizes the image.""" + + def __init__( + self, *, encoded_sample_type=None, raw_batch_type=None, batch_type=None, crop_height=256, temporal_crop=49 + ): + super().__init__(encoded_sample_type=encoded_sample_type, raw_batch_type=raw_batch_type, batch_type=batch_type) + + crop_sizes = get_crop_size_info(crop_height) + self.spatial_crop = RandomCrop(input_keys=[VIDEO_KEY], args={"size": crop_sizes}) + self.temporal_crop = TemporalRandomCrop(temporal_crop) + self.normalize = Normalize(input_keys=[VIDEO_KEY], args={"mean": 0.5, "std": 0.5}) + + def encode_sample(self, sample): + """ + Encode a single image sample by cropping and shifting its values. + + Args: + sample: An image sample. + + Returns: + The transformed image sample. + """ + sample = super().encode_sample(sample) + sample.image.frames = rearrange(sample.image.frames, "t c h w -> c t h w") + + mask_t = torch.ones_like( + sample.image.frames, requires_grad=False, dtype=torch.bfloat16, device=sample.image.frames.device + ) + + data = {VIDEO_KEY: sample.image.frames, MASK_KEY: mask_t, "aspect_ratio": "1,1"} + data = self.spatial_crop(data) + begin_index, end_index = self.temporal_crop(data[VIDEO_KEY].shape[1]) + data[VIDEO_KEY] = data[VIDEO_KEY][:, begin_index:end_index] + data = self.normalize(data) + data[VIDEO_KEY] = data[VIDEO_KEY].to(torch.bfloat16) + return data + + +@run.cli.factory(target=llm.train) +def train_tokenizer() -> run.Partial: + return run.Partial( + llm.train, + model=run.Config( + TokenizerModel, + jit_ckpt_pth=None, + model="Cosmos-1.0-Tokenizer-CV8x8x8", + ), + data=run.Config( + DiffusionDataModule, + path=None, + task_encoder=run.Config(ImageTaskEncoder), + global_batch_size=8, + micro_batch_size=1, + num_workers=1, + ), + trainer=run.Config( + nl.Trainer, + devices="auto", + num_nodes=int(os.environ.get("SLURM_NNODES", 1)), + accelerator="gpu", + strategy="ddp_find_unused_parameters_true", + num_sanity_val_steps=0, + limit_val_batches=1, + val_check_interval=100, + max_epochs=10000, + precision="bf16", + logger=WandbLogger(project="cosmos-tokenizer") if "WANDB_API_KEY" in os.environ else None, + log_every_n_steps=1, + use_distributed_sampler=False, + callbacks=[ + run.Config( + ModelCheckpoint, + monitor="global_step", + filename="{global_step}", + every_n_train_steps=100, + save_top_k=3, + mode="max", + always_save_context=False, + save_context_on_train_end=False, + ), + run.Config(PreemptionCallback), + run.Config(TimingCallback), + ], + ), + optim=run.Config( + PytorchOptimizerModule, + optimizer_fn=run.Partial( + torch.optim.AdamW, + lr=1e-4, + betas=(0.5, 0.999), + eps=1e-8, + weight_decay=0.01, + fused=True, + ), + ), + tokenizer=None, + resume=run.Config( + nl.AutoResume, + resume_if_exists=True, + resume_ignore_no_checkpoint=True, + resume_past_end=True, + ), + model_transform=None, + ) + + +if __name__ == "__main__": + run.cli.main(llm.train, default_factory=train_tokenizer) diff --git a/nemo_vfm/sparse_attention/README.md b/nemo_vfm/sparse_attention/README.md new file mode 100644 index 00000000..bb10a88f --- /dev/null +++ b/nemo_vfm/sparse_attention/README.md @@ -0,0 +1,69 @@ +# Sparse Attention + +## 1. Initialize Repositories + +### 1.1. Clone Repositories + +```bash +git clone https://gitlab-master.nvidia.com/sgovande/sparse-attention && cd sparse-attention +git clone https://github.com/nvidia-cosmos/cosmos-predict1 +``` + +### 1.2. Initialize Cosmos Dependencies + +Follow the instructions in [cosmos-predict1/INSTALL.md](https://github.com/nvidia-cosmos/cosmos-predict1/blob/main/INSTALL.md) to set your environment up. You will need to install dependencies for both the **Inference** and the **Post-Training** sections. + + +### 1.3. Install Sparse Attention + +```bash +pip install git+https://github.com/sandyresearch/chipmunk@master --no-build-isolation +pip install flash-attn==2.4.2 --no-build-isolation +``` + +### 1.4. Update Inference Script +Patch the inference script to activate sparse attention. +```bash +mv accel-config.yml cosmos-predict1/accel-config.yml +mv sparse_attn.py cosmos-predict1/sparse_attn.py +sed -i '1i from sparse_attn import setup_sparse_attn' cosmos-predict1/cosmos_predict1/diffusion/inference/text2world.py +sed -i '2i setup_sparse_attn("accel-config.yml")' cosmos-predict1/cosmos_predict1/diffusion/inference/text2world.py +``` + +## 2. Modify the Sparsity Configuration + +The sparsity configuration file is located in [`accel-config.yml`](../../../accel-config.yml). There is a certain set of parameters controlling the sparsity level to modulate the speed-quality tradeoff, and there are other parameters for the overall structure of the model. + +### 2.1. Configure the Speed-Quality Tradeoff + +On 75% sparsity (the recommended default), you can expect to see a generation time of 201 seconds on a H100 GPU. This is over 2.0x down from the original generation time of 410 seconds. There are two key hyperparameters to control the level of sparsity: + +- **Main Sparsity Parameter**: `attn.top_keys`: This affects how many keys/values are selected for every query, +- **Other Sparsity Parameters**: `attn.full_step_every`: This affects how often we interleave a dense step with the sparse steps. + +### 2.2. Configure the Model Shapes + +Ensure that the `model_config` key in the configuration file aligns with the characteristics of the model, including the model's latent vector shape **(W, H, D)**, the number of heads, and the number of layers. + +## 3. Generate a video. + +Run the `text2world.py` script to generate a video. + +```bash +export NVTE_FUSED_ATTN=0 +export NVTE_UNFUSED_ATTN=0 +export NVTE_FLASH_ATTN=1 +``` + + +```bash +export PROMPT="Hello World" +cd cosmos-predict1 +CUDA_HOME=$CONDA_PREFIX PYTHONPATH=$(pwd) python cosmos_predict1/diffusion/inference/text2world.py \ + --checkpoint_dir checkpoints \ + --diffusion_transformer_dir Cosmos-Predict1-7B-Text2World \ + --prompt "${PROMPT}" \ + --video_save_name diffusion-text2world-7b \ + --disable_prompt_upsampler \ + --disable_guardrail +``` \ No newline at end of file diff --git a/nemo_vfm/sparse_attention/accel-config.yml b/nemo_vfm/sparse_attention/accel-config.yml new file mode 100644 index 00000000..3fa03862 --- /dev/null +++ b/nemo_vfm/sparse_attention/accel-config.yml @@ -0,0 +1,41 @@ +num_model_invocations_per_inference_step: 1 + +model_config: + name: cosmos + + latent_vector_shape: + width: 80 + height: 44 + depth: 16 + + num_heads: 32 + num_layers: 28 + + context_parallel_size: 1 + tensor_parallel_size: 1 + num_steps: 70 + +attn: + is_enabled: true + top_keys: 0.25 + random_keys: 0.01 + + first_n_dense_layers: 2 + recompute_mask: true + should_compress_indices: true + + # If schedule is not None, will override full_step_every + full_step_schedule: !!set + ? 0 + ? 1 + ? 10 + ? 40 + # full_step_every: 10 + + pad_qkv_before_kernel: true + counts_multiple_of: 128 + +offloading: + global_disable_offloading: true +step_caching: + is_enabled: false \ No newline at end of file diff --git a/nemo_vfm/sparse_attention/kernels/tl_base_attention.py b/nemo_vfm/sparse_attention/kernels/tl_base_attention.py new file mode 100644 index 00000000..360a216e --- /dev/null +++ b/nemo_vfm/sparse_attention/kernels/tl_base_attention.py @@ -0,0 +1,883 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +import triton +import triton.language as tl +from torch.nn import functional as F + + +# DEVICE = triton.runtime.driver.active.get_active_torch_device() +DEVICE = torch.device("cuda") + + +@triton.jit +def _attn_fwd_inner( + acc, + l_i, + m_i, + q, # + K_block_ptr, + V_block_ptr, # + start_m, + qk_scale, # + seqlen, + BLOCK_M: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_N: tl.constexpr, # + STAGE: tl.constexpr, + offs_m: tl.constexpr, + offs_n: tl.constexpr, # + N_CTX: tl.constexpr, + fp8_v: tl.constexpr, +): + # range of values handled by this stage + if STAGE == 1: + lo, hi = 0, start_m * BLOCK_M + elif STAGE == 2: + lo, hi = start_m * BLOCK_M, (start_m + 1) * BLOCK_M + lo = tl.multiple_of(lo, BLOCK_M) + # causal = False + else: + lo, hi = 0, N_CTX + K_block_ptr = tl.advance(K_block_ptr, (0, lo)) + V_block_ptr = tl.advance(V_block_ptr, (lo, 0)) + # loop over k, v and update accumulator + for start_n in range(lo, hi, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(K_block_ptr) + # qk = tl.dot(q, k) + tl.where(start_n + offs_n[None, :] < seqlen, 0, -1.0e6) + qk = tl.dot(q, k) + # qk = tl.where(start_n + offs_n[None, :] < 4592, qk, -1.0e6) + if STAGE == 2: + mask = offs_m[:, None] >= (start_n + offs_n[None, :]) + qk = qk * qk_scale + tl.where(mask, 0, -1.0e6) + m_ij = tl.maximum(m_i, tl.max(qk, 1)) + qk -= m_ij[:, None] + else: + m_ij = tl.maximum(m_i, tl.max(qk, 1) * qk_scale) + qk = qk * qk_scale - m_ij[:, None] + # qk = qk * qk_scale - m_ij[:, None] + tl.where(start_n + offs_n[None, :] < 4592, 0, -1.0e6) + # qk = tl.where(start_n + offs_n[None, :] < 4592, qk, -1.0e6) + p = tl.math.exp2(qk) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + alpha = tl.math.exp2(m_i - m_ij) + l_i = l_i * alpha + l_ij + # -- update output accumulator -- + acc = acc * alpha[:, None] + # update acc + v = tl.load(V_block_ptr) + # v = tl.where(start_n + offs_n[:, None] < 4592, v, 0).to(tl.bfloat16) + if fp8_v: + p = p.to(tl.float8e5) + else: + p = p.to(tl.bfloat16) + acc = tl.dot(p, v, acc) + # update m_i and l_i + m_i = m_ij + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + return acc, l_i, m_i + + +# We don't run auto-tuning every time to keep the tutorial fast. Keeping +# the code below and commenting out the equivalent parameters is convenient for +# re-tuning. +configs = [ + triton.Config({"BLOCK_M": BM, "BLOCK_N": BN}, num_stages=s, num_warps=w) + # for BM in [64, 128]\ + # for BN in [32, 64]\ + for BM in [64] + for BN in [64] + for s in ([3, 4, 7]) + for w in [4, 8] +] + + +def keep(conf): + BLOCK_M = conf.kwargs["BLOCK_M"] + BLOCK_N = conf.kwargs["BLOCK_N"] + if BLOCK_M * BLOCK_N < 128 * 128 and conf.num_warps == 8: + return False + return True + + +@triton.autotune(list(filter(keep, configs)), key=["N_CTX", "HEAD_DIM"]) +@triton.jit +def _attn_fwd( + Q, + K, + V, + sm_scale, + M, + L, + Out, + seqlen, # + stride_qz, + stride_qh, + stride_qm, + stride_qk, # + stride_kz, + stride_kh, + stride_kn, + stride_kk, # + stride_vz, + stride_vh, + stride_vk, + stride_vn, # + stride_oz, + stride_oh, + stride_om, + stride_on, # + Z, + H, + N_CTX, # + HEAD_DIM: tl.constexpr, # + BLOCK_M: tl.constexpr, # + BLOCK_N: tl.constexpr, # + STAGE: tl.constexpr, # +): + tl.static_assert(BLOCK_N <= HEAD_DIM) + start_m = tl.program_id(0) + off_hz = tl.program_id(1) + off_z = off_hz // H + off_h = off_hz % H + qvk_offset = off_z.to(tl.int64) * stride_qz + off_h.to(tl.int64) * stride_qh + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + offs_headsize = tl.arange(0, HEAD_DIM) + + # block pointers + Q_block_ptr = Q + qvk_offset + offs_m[:, None] * stride_qm + offs_headsize[None, :] * stride_qk + v_order: tl.constexpr = (0, 1) if V.dtype.element_ty == tl.float8e5 else (1, 0) + V_block_ptr = tl.make_block_ptr( + base=V + qvk_offset, + shape=(N_CTX, HEAD_DIM), + strides=(stride_vk, stride_vn), + offsets=(0, 0), + block_shape=(BLOCK_N, HEAD_DIM), + order=v_order, + ) + K_block_ptr = tl.make_block_ptr( + base=K + qvk_offset, + shape=(HEAD_DIM, N_CTX), + strides=(stride_kk, stride_kn), + offsets=(0, 0), + block_shape=(HEAD_DIM, BLOCK_N), + order=(0, 1), + ) + # O_block_ptr = tl.make_block_ptr( + # base=Out + qvk_offset, + # shape=(N_CTX, HEAD_DIM), + # strides=(stride_om, stride_on), + # offsets=(start_m * BLOCK_M, 0), + # block_shape=(BLOCK_M, HEAD_DIM), + # order=(1, 0), + # ) + offs_o = (start_m * BLOCK_M + tl.arange(0, BLOCK_M)[:, None]) * stride_om + tl.arange(0, HEAD_DIM)[ + None, : + ] * stride_on + O_ptrs = Out + qvk_offset + offs_o + + # initialize offsets + # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + 1.0 + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + # load scales + qk_scale = sm_scale + qk_scale *= 1.44269504 # 1/log(2) + # load q: it will stay in SRAM throughout + q = tl.load(Q_block_ptr, mask=offs_m[:, None] < seqlen) + # stage 1: off-band + # For causal = True, STAGE = 3 and _attn_fwd_inner gets 1 as its STAGE + # For causal = False, STAGE = 1, and _attn_fwd_inner gets 3 as its STAGE + if STAGE & 1: + acc, l_i, m_i = _attn_fwd_inner( + acc, + l_i, + m_i, + q, + K_block_ptr, + V_block_ptr, # + start_m, + qk_scale, + seqlen, # + BLOCK_M, + HEAD_DIM, + BLOCK_N, # + 4 - STAGE, + offs_m, + offs_n, + N_CTX, + V.dtype.element_ty == tl.float8e5, # + ) + # stage 2: on-band + if STAGE & 2: + # barrier makes it easier for compielr to schedule the + # two loops independently + acc, l_i, m_i = _attn_fwd_inner( + acc, + l_i, + m_i, + q, + K_block_ptr, + V_block_ptr, # + start_m, + qk_scale, + seqlen, # + BLOCK_M, + HEAD_DIM, + BLOCK_N, # + 2, + offs_m, + offs_n, + N_CTX, + V.dtype.element_ty == tl.float8e5, # + ) + # epilogue + m_i += tl.math.log2(l_i) + acc = acc / l_i[:, None] + m_ptrs = M + off_hz * N_CTX + offs_m + l_ptrs = L + off_hz * N_CTX + offs_m + tl.store(m_ptrs, m_i, mask=offs_m < seqlen) + tl.store(l_ptrs, l_i, mask=offs_m < seqlen) + # tl.store(m_ptrs, m_i) + # tl.store(l_ptrs, l_i) + # tl.store(O_block_ptr, acc.to(Out.type.element_ty)) + tl.store(O_ptrs, acc.to(Out.type.element_ty), mask=offs_m[:, None] < seqlen) + # tl.store(O_ptrs, acc.to(Out.type.element_ty)) + + +@triton.jit +def _attn_bwd_preprocess( + O, + DO, # + Delta, # + Z, + H, + N_CTX, # + BLOCK_M: tl.constexpr, + HEAD_DIM: tl.constexpr, # +): + off_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + off_hz = tl.program_id(1) + off_n = tl.arange(0, HEAD_DIM) + # load + o = tl.load(O + off_hz * HEAD_DIM * N_CTX + off_m[:, None] * HEAD_DIM + off_n[None, :]) + do = tl.load(DO + off_hz * HEAD_DIM * N_CTX + off_m[:, None] * HEAD_DIM + off_n[None, :]).to(tl.float32) + delta = tl.sum(o * do, axis=1) + # write-back + tl.store(Delta + off_hz * N_CTX + off_m, delta) + + +# The main inner-loop logic for computing dK and dV. +@triton.jit +def _attn_bwd_dkdv( + dk, + dv, # + Q, + k, + v, + sm_scale, # + DO, # + M, + D, # + # shared by Q/K/V/DO. + stride_tok, + stride_d, # + H, + N_CTX, + BLOCK_M1: tl.constexpr, # + BLOCK_N1: tl.constexpr, # + HEAD_DIM: tl.constexpr, # + # Filled in by the wrapper. + start_n, + start_m, + num_steps, # + MASK: tl.constexpr, +): + offs_m = start_m + tl.arange(0, BLOCK_M1) + offs_n = start_n + tl.arange(0, BLOCK_N1) + offs_k = tl.arange(0, HEAD_DIM) + qT_ptrs = Q + offs_m[None, :] * stride_tok + offs_k[:, None] * stride_d + do_ptrs = DO + offs_m[:, None] * stride_tok + offs_k[None, :] * stride_d + # BLOCK_N1 must be a multiple of BLOCK_M1, otherwise the code wouldn't work. + tl.static_assert(BLOCK_N1 % BLOCK_M1 == 0) + curr_m = start_m + step_m = BLOCK_M1 + for blk_idx in range(num_steps): + qT = tl.load(qT_ptrs) + # Load m before computing qk to reduce pipeline stall. + offs_m = curr_m + tl.arange(0, BLOCK_M1) + m = tl.load(M + offs_m) + qkT = tl.dot(k, qT) + pT = tl.math.exp2(qkT - m[None, :]) + # Autoregressive masking. + if MASK: + mask = offs_m[None, :] >= offs_n[:, None] + pT = tl.where(mask, pT, 0.0) + do = tl.load(do_ptrs) + # Compute dV. + ppT = pT + ppT = ppT.to(tl.bfloat16) + dv += tl.dot(ppT, do) + # D (= delta) is pre-divided by ds_scale. + Di = tl.load(D + offs_m) + # Compute dP and dS. + dpT = tl.dot(v, tl.trans(do)).to(tl.float32) + dsT = pT * (dpT - Di[None, :]) + dsT = dsT.to(tl.bfloat16) + dk += tl.dot(dsT, tl.trans(qT)) + # Increment pointers. + curr_m += step_m + qT_ptrs += step_m * stride_tok + do_ptrs += step_m * stride_tok + return dk, dv + + +# the main inner-loop logic for computing dQ +@triton.jit +def _attn_bwd_dq( + dq, + q, + K, + V, # + do, + m, + D, + # shared by Q/K/V/DO. + stride_tok, + stride_d, # + H, + N_CTX, # + BLOCK_M2: tl.constexpr, # + BLOCK_N2: tl.constexpr, # + HEAD_DIM: tl.constexpr, + # Filled in by the wrapper. + start_m, + start_n, + num_steps, # + MASK: tl.constexpr, +): + offs_m = start_m + tl.arange(0, BLOCK_M2) + offs_n = start_n + tl.arange(0, BLOCK_N2) + offs_k = tl.arange(0, HEAD_DIM) + kT_ptrs = K + offs_n[None, :] * stride_tok + offs_k[:, None] * stride_d + vT_ptrs = V + offs_n[None, :] * stride_tok + offs_k[:, None] * stride_d + # D (= delta) is pre-divided by ds_scale. + Di = tl.load(D + offs_m) + # BLOCK_M2 must be a multiple of BLOCK_N2, otherwise the code wouldn't work. + tl.static_assert(BLOCK_M2 % BLOCK_N2 == 0) + curr_n = start_n + step_n = BLOCK_N2 + for blk_idx in range(num_steps): + kT = tl.load(kT_ptrs) + vT = tl.load(vT_ptrs) + qk = tl.dot(q, kT) + p = tl.math.exp2(qk - m) + # Autoregressive masking. + if MASK: + offs_n = curr_n + tl.arange(0, BLOCK_N2) + mask = offs_m[:, None] >= offs_n[None, :] + p = tl.where(mask, p, 0.0) + # Compute dP and dS. + dp = tl.dot(do, vT).to(tl.float32) + ds = p * (dp - Di[:, None]) + ds = ds.to(tl.bfloat16) + # Compute dQ. + # NOTE: We need to de-scale dq in the end, because kT was pre-scaled. + dq += tl.dot(ds, tl.trans(kT)) + # Increment pointers. + curr_n += step_n + kT_ptrs += step_n * stride_tok + vT_ptrs += step_n * stride_tok + return dq + + +@triton.jit +def _attn_bwd( + Q, + K, + V, + sm_scale, # + DO, # + DQ, + DK, + DV, # + M, + D, + # shared by Q/K/V/DO. + stride_z, + stride_h, + stride_tok, + stride_d, # + H, + N_CTX, # + BLOCK_M1: tl.constexpr, # + BLOCK_N1: tl.constexpr, # + BLOCK_M2: tl.constexpr, # + BLOCK_N2: tl.constexpr, # + BLK_SLICE_FACTOR: tl.constexpr, # + HEAD_DIM: tl.constexpr, +): + LN2: tl.constexpr = 0.6931471824645996 # = ln(2) + + bhid = tl.program_id(2) + off_chz = (bhid * N_CTX).to(tl.int64) + adj = (stride_h * (bhid % H) + stride_z * (bhid // H)).to(tl.int64) + pid = tl.program_id(0) + + # offset pointers for batch/head + Q += adj + K += adj + V += adj + DO += adj + DQ += adj + DK += adj + DV += adj + M += off_chz + D += off_chz + + # load scales + offs_k = tl.arange(0, HEAD_DIM) + + start_n = pid * BLOCK_N1 + start_m = start_n + + MASK_BLOCK_M1: tl.constexpr = BLOCK_M1 // BLK_SLICE_FACTOR + offs_n = start_n + tl.arange(0, BLOCK_N1) + + dv = tl.zeros([BLOCK_N1, HEAD_DIM], dtype=tl.float32) + dk = tl.zeros([BLOCK_N1, HEAD_DIM], dtype=tl.float32) + + # load K and V: they stay in SRAM throughout the inner loop. + k = tl.load(K + offs_n[:, None] * stride_tok + offs_k[None, :] * stride_d) + v = tl.load(V + offs_n[:, None] * stride_tok + offs_k[None, :] * stride_d) + + num_steps = BLOCK_N1 // MASK_BLOCK_M1 + + dk, dv = _attn_bwd_dkdv( + dk, + dv, # + Q, + k, + v, + sm_scale, # + DO, # + M, + D, # + stride_tok, + stride_d, # + H, + N_CTX, # + MASK_BLOCK_M1, + BLOCK_N1, + HEAD_DIM, # + start_n, + start_m, + num_steps, # + MASK=True, # + ) + + start_m += num_steps * MASK_BLOCK_M1 + num_steps = (N_CTX - start_m) // BLOCK_M1 + + # Compute dK and dV for non-masked blocks. + dk, dv = _attn_bwd_dkdv( # + dk, + dv, # + Q, + k, + v, + sm_scale, # + DO, # + M, + D, # + stride_tok, + stride_d, # + H, + N_CTX, # + BLOCK_M1, + BLOCK_N1, + HEAD_DIM, # + start_n, + start_m, + num_steps, # + MASK=False, # + ) + + dv_ptrs = DV + offs_n[:, None] * stride_tok + offs_k[None, :] * stride_d + tl.store(dv_ptrs, dv) + + # Write back dK. + dk *= sm_scale + dk_ptrs = DK + offs_n[:, None] * stride_tok + offs_k[None, :] * stride_d + tl.store(dk_ptrs, dk) + + # THIS BLOCK DOES DQ: + start_m = pid * BLOCK_M2 + end_n = start_m + BLOCK_M2 + + MASK_BLOCK_N2: tl.constexpr = BLOCK_N2 // BLK_SLICE_FACTOR + offs_m = start_m + tl.arange(0, BLOCK_M2) + + q = tl.load(Q + offs_m[:, None] * stride_tok + offs_k[None, :] * stride_d) + dq = tl.zeros([BLOCK_M2, HEAD_DIM], dtype=tl.float32) + do = tl.load(DO + offs_m[:, None] * stride_tok + offs_k[None, :] * stride_d) + + m = tl.load(M + offs_m) + m = m[:, None] + + # Compute dQ for masked (diagonal) blocks. + # NOTE: This code scans each row of QK^T backward (from right to left, + # but inside each call to _attn_bwd_dq, from left to right), but that's + # not due to anything important. I just wanted to reuse the loop + # structure for dK & dV above as much as possible. + num_steps = BLOCK_M2 // MASK_BLOCK_N2 + dq = _attn_bwd_dq( + dq, + q, + K, + V, # + do, + m, + D, # + stride_tok, + stride_d, # + H, + N_CTX, # + BLOCK_M2, + MASK_BLOCK_N2, + HEAD_DIM, # + start_m, + end_n - num_steps * MASK_BLOCK_N2, + num_steps, # + MASK=True, # + ) + end_n -= num_steps * MASK_BLOCK_N2 + # stage 2 + num_steps = end_n // BLOCK_N2 + dq = _attn_bwd_dq( + dq, + q, + K, + V, # + do, + m, + D, # + stride_tok, + stride_d, # + H, + N_CTX, # + BLOCK_M2, + BLOCK_N2, + HEAD_DIM, # + start_m, + end_n - num_steps * BLOCK_N2, + num_steps, # + MASK=False, # + ) + # Write back dQ. + dq_ptrs = DQ + offs_m[:, None] * stride_tok + offs_k[None, :] * stride_d + dq *= LN2 + tl.store(dq_ptrs, dq) + + +class _attention(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, v, causal, sm_scale): + # shape constraints + HEAD_DIM_Q, HEAD_DIM_K = q.shape[-1], k.shape[-1] + # when v is in float8_e5m2 it is transposed. + HEAD_DIM_V = v.shape[-1] + assert HEAD_DIM_Q == HEAD_DIM_K and HEAD_DIM_K == HEAD_DIM_V + assert HEAD_DIM_K in {16, 32, 64, 128, 256} + o = torch.empty_like(q) + stage = 3 if causal else 1 + extra_kern_args = {} + grid = lambda args: (triton.cdiv(q.shape[2], args["BLOCK_M"]), q.shape[0] * q.shape[1], 1) + M = torch.empty((q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32) + L = torch.empty((q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32) + seqlen = q.shape[2] + _attn_fwd[grid]( + q, + k, + v, + sm_scale, + M, + L, + o, + seqlen, # + q.stride(0), + q.stride(1), + q.stride(2), + q.stride(3), # + k.stride(0), + k.stride(1), + k.stride(2), + k.stride(3), # + v.stride(0), + v.stride(1), + v.stride(2), + v.stride(3), # + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), # + q.shape[0], + q.shape[1], # + N_CTX=q.shape[2], # + HEAD_DIM=HEAD_DIM_K, # + STAGE=stage, # + **extra_kern_args, + ) + + return o, M, L + + @staticmethod + def backward(ctx, do): + q, k, v, o, M = ctx.saved_tensors + assert do.is_contiguous() + assert q.stride() == k.stride() == v.stride() == o.stride() == do.stride() + dq = torch.empty_like(q) + dk = torch.empty_like(k) + dv = torch.empty_like(v) + BATCH, N_HEAD, N_CTX = q.shape[:3] + PRE_BLOCK = 128 + NUM_WARPS, NUM_STAGES = 4, 5 + BLOCK_M1, BLOCK_N1, BLOCK_M2, BLOCK_N2 = 32, 128, 128, 32 + BLK_SLICE_FACTOR = 2 + RCP_LN2 = 1.4426950408889634 # = 1.0 / ln(2) + arg_k = k + arg_k = arg_k * (ctx.sm_scale * RCP_LN2) + PRE_BLOCK = 128 + assert N_CTX % PRE_BLOCK == 0 + pre_grid = (N_CTX // PRE_BLOCK, BATCH * N_HEAD) + delta = torch.empty_like(M) + _attn_bwd_preprocess[pre_grid]( + o, + do, # + delta, # + BATCH, + N_HEAD, + N_CTX, # + BLOCK_M=PRE_BLOCK, + HEAD_DIM=ctx.HEAD_DIM, # + ) + grid = (N_CTX // BLOCK_N1, 1, BATCH * N_HEAD) + _attn_bwd[grid]( + q, + arg_k, + v, + ctx.sm_scale, + do, + dq, + dk, + dv, # + M, + delta, # + q.stride(0), + q.stride(1), + q.stride(2), + q.stride(3), # + N_HEAD, + N_CTX, # + BLOCK_M1=BLOCK_M1, + BLOCK_N1=BLOCK_N1, # + BLOCK_M2=BLOCK_M2, + BLOCK_N2=BLOCK_N2, # + BLK_SLICE_FACTOR=BLK_SLICE_FACTOR, # + HEAD_DIM=ctx.HEAD_DIM, # + num_warps=NUM_WARPS, # + num_stages=NUM_STAGES, # + ) + + return dq, dk, dv, None, None + + +attention = _attention.apply + + +# @pytest.mark.parametrize("Z, H, N_CTX, HEAD_DIM", [(1, 2, 1024, 64)]) +# @pytest.mark.parametrize("causal", [False]) +def test_op(Z, H, N_CTX, HEAD_DIM, causal, dtype=torch.bfloat16): + torch.manual_seed(20) + q = torch.empty((Z, H, N_CTX, HEAD_DIM), dtype=dtype, device=DEVICE).normal_(mean=0.0, std=0.5).requires_grad_() + k = torch.empty((Z, H, N_CTX, HEAD_DIM), dtype=dtype, device=DEVICE).normal_(mean=0.0, std=0.5).requires_grad_() + v = torch.empty((Z, H, N_CTX, HEAD_DIM), dtype=dtype, device=DEVICE).normal_(mean=0.0, std=0.5).requires_grad_() + # qkv = torch.load('../qkv-nans-step-0-layer-4.pt') + # q, k, v = qkv[0], qkv[1], qkv[2] + # q.requires_grad_() + # k.requires_grad_() + # v.requires_grad_() + print(f"q: {q.shape}") + sm_scale = 1 / HEAD_DIM**0.5 + dout = torch.randn_like(q) + # reference implementation + M = torch.tril(torch.ones((N_CTX, N_CTX), device=DEVICE)) + p = torch.matmul(q, k.transpose(2, 3)) * sm_scale + p *= 1.44269504 # 1/log(2) + if causal: + p[:, :, M == 0] = float("-inf") + + p = torch.softmax(p.float(), dim=-1).to(dtype) + # p = p.float() + # refm = torch.max(p, dim=-1, keepdim=True)[0] + # num = torch.exp2(p - refm) + # refl = torch.sum(num, dim=-1, keepdim=True) + # p = num / refl + # p = p.to(dtype) + # refm = refm.squeeze(-1) + # refl = refl.squeeze(-1) + + # p = torch.exp(p) + ref_out = torch.matmul(p, v) + # ref_out.backward(dout) + # ref_dv, v.grad = v.grad.clone(), None + # ref_dk, k.grad = k.grad.clone(), None + # ref_dq, q.grad = q.grad.clone(), None + # triton implementation + tri_out, m, l = attention(q, k, v, causal, sm_scale) + tri_out = tri_out.to(dtype) + # tri_out.backward(dout) + # tri_dv, v.grad = v.grad.clone(), None + # tri_dk, k.grad = k.grad.clone(), None + # tri_dq, q.grad = q.grad.clone(), None + spref = F.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=causal) + print(f"spref: {spref[0, -1, -130:-110]}") + print(f"tri_out: {tri_out[0, -1, -130:-110]}") + print(f"diff: {(spref - tri_out)[0, -1, -130:-110].sort(dim=-1).values}") + print(f"ref_out: {ref_out[0, -1, -130:-110]}") + # compare + print(f"diff max: {torch.max(torch.abs(spref - tri_out))}") + # print(f'diff inds: {torch.argmax(torch.abs(spref - tri_out), dim=-1)}') + print(f"spda to triton dist: {torch.dist(spref, tri_out)}") + print(f"spda to ref dist: {torch.dist(spref, ref_out)}") + assert torch.allclose(spref, tri_out, atol=1e-3, rtol=0) + # print(f'ref m: {refm}') + # print(f'triton m: {m}') + # assert torch.allclose(refm, m, atol=1e-2, rtol=0) + # print(f'ref l: {refl}') + # print(f'triton l: {l}') + # assert torch.allclose(refl, l, atol=1, rtol=0) + rtol = 0.0 + # Relative tolerance workaround for known hardware limitation of MI200 GPU. + # For details see https://pytorch.org/docs/stable/notes/numerical_accuracy.html#reduced-precision-fp16-and-bf16-gemms-and-convolutions-on-amd-instinct-mi200-devices + if torch.version.hip is not None and triton.runtime.driver.active.get_current_target().arch == "gfx90a": + rtol = 1e-2 + # assert torch.allclose(ref_dv, tri_dv, atol=1e-2, rtol=rtol) + # assert torch.allclose(ref_dk, tri_dk, atol=1e-2, rtol=rtol) + # assert torch.allclose(ref_dq, tri_dq, atol=1e-2, rtol=rtol) + + +try: + from flash_attn.flash_attn_interface import flash_attn_qkvpacked_func as flash_attn_func + + HAS_FLASH = True +except BaseException: + HAS_FLASH = False + +# TORCH_HAS_FP8 = hasattr(torch, 'float8_e5m2') +TORCH_HAS_FP8 = False +BATCH, N_HEADS, HEAD_DIM = 1, 24, 128 +# vary seq length for fixed head and batch=4 +configs = [] +for mode in ["fwd"]: + for causal in [False]: + if mode == "bwd" and not causal: + continue + configs.append( + triton.testing.Benchmark( + x_names=["N_CTX"], + x_vals=[4096], + line_arg="provider", + line_vals=["triton-fp16", "torch"] + + (["triton-fp8"] if TORCH_HAS_FP8 else []) + + (["flash"] if HAS_FLASH else []), + line_names=["Triton [FP16]", "Torch [FP16]"] + + (["Triton [FP8]"] if TORCH_HAS_FP8 else []) + + (["Flash-2"] if HAS_FLASH else []), + styles=[("red", "-"), ("blue", "-"), ("green", "-")], + ylabel="TFLOPS", + plot_name=f"fused-attention-batch{BATCH}-head{N_HEADS}-d{HEAD_DIM}-{mode}-causal={causal}", + args={ + "H": N_HEADS, + "BATCH": BATCH, + "HEAD_DIM": HEAD_DIM, + "mode": mode, + "causal": causal, + }, + ) + ) + + +@triton.testing.perf_report(configs) +def bench_flash_attention(BATCH, H, N_CTX, HEAD_DIM, causal, mode, provider, device=DEVICE): + assert mode in ["fwd", "bwd"] + dtype = torch.bfloat16 + if "triton" in provider: + q = torch.randn((BATCH, H, N_CTX, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + k = torch.randn((BATCH, H, N_CTX, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + v = torch.randn((BATCH, H, N_CTX, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + if mode == "fwd" and "fp8" in provider: + q = q.to(torch.float8_e5m2) + k = k.to(torch.float8_e5m2) + v = v.permute(0, 1, 3, 2).contiguous() + v = v.permute(0, 1, 3, 2) + v = v.to(torch.float8_e5m2) + sm_scale = 1.3 + fn = lambda: attention(q, k, v, causal, sm_scale) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) + ms = triton.testing.do_bench(fn) + if provider == "torch": + q = torch.randn((BATCH, H, N_CTX, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + k = torch.randn((BATCH, H, N_CTX, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + v = torch.randn((BATCH, H, N_CTX, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + fn = lambda: F.scaled_dot_product_attention(q, k, v) + if mode == "bwd": + o, m, l = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) + ms = triton.testing.do_bench(fn) + if provider == "flash": + qkv = torch.randn((BATCH, N_CTX, 3, H, HEAD_DIM), dtype=dtype, device=device, requires_grad=True) + fn = lambda: flash_attn_func(qkv, causal=causal) + if mode == "bwd": + o, m, l = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) + ms = triton.testing.do_bench(fn) + flops_per_matmul = 2.0 * BATCH * H * N_CTX * N_CTX * HEAD_DIM + total_flops = 2 * flops_per_matmul + if causal: + total_flops *= 0.5 + if mode == "bwd": + total_flops *= 2.5 # 2.0(bwd) + 0.5(recompute) + return total_flops * 1e-12 / (ms * 1e-3) + + +if __name__ == "__main__": + # only works on post-Ampere GPUs right now + # for i in range(10): + # print(f'run {i}') + # test_op(1, 24, 34 * 128 + 0, 128, False) + bench_flash_attention.run(print_data=True, show_plots=False) diff --git a/nemo_vfm/sparse_attention/kernels/tl_blocksum_attention.py b/nemo_vfm/sparse_attention/kernels/tl_blocksum_attention.py new file mode 100644 index 00000000..9658b3b5 --- /dev/null +++ b/nemo_vfm/sparse_attention/kernels/tl_blocksum_attention.py @@ -0,0 +1,521 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import torch +import triton +import triton.language as tl +from torch.nn import functional as F + + +DEVICE = "cuda" + + +cdiv = lambda a, b: (a + b - 1) // b + +# We don't run auto-tuning every time to keep the tutorial fast. Keeping +# the code below and commenting out the equivalent parameters is convenient for +# re-tuning. +configs = [ + triton.Config({"BLOCK_M": BM, "BLOCK_N": BN}, num_stages=s, num_warps=w) + for BM in [64] + for BN in [64] + for s in ([3, 4, 7]) + for w in [4, 8] +] + + +def keep(conf): + BLOCK_M = conf.kwargs["BLOCK_M"] + BLOCK_N = conf.kwargs["BLOCK_N"] + if BLOCK_M * BLOCK_N < 128 * 128 and conf.num_warps == 8: + return False + return True + + +@triton.jit +def _full_attn_fwd_inner( + acc, + l_i, + m_i, + q, # + prev_maxes_final_ptrs, # + prev_normalization_final_ptrs, # + blocksums_ptrs, + softmax_stride_b, + softmax_stride_h, + softmax_stride_n, # + blocksums_stride_b, + blocksums_stride_h, + blocksums_stride_m, + blocksums_stride_n, + K_block_ptr, + V_block_ptr, # + start_m, + qk_scale, + seqlen, # + H, # + BLOCK_M: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_N: tl.constexpr, # + STAGE: tl.constexpr, + offs_m: tl.constexpr, + offs_n: tl.constexpr, # + N_CTX: tl.constexpr, + fp8_v: tl.constexpr, +): + # non-causal full attention + lo, hi = 0, N_CTX + K_block_ptr = tl.advance(K_block_ptr, (0, lo)) + V_block_ptr = tl.advance(V_block_ptr, (lo, 0)) + off_hb = tl.program_id(1) + off_b = off_hb // H + off_h = off_hb % H + softmax_data_offset = ( + off_b.to(tl.int64) * softmax_stride_b + + off_h.to(tl.int64) * softmax_stride_h + + (start_m * BLOCK_M + tl.arange(0, BLOCK_M)) * softmax_stride_n + ) + # blocksums_ptrs += off_b.to(tl.int64) * blocksums_stride_b + off_h.to(tl.int64) * blocksums_stride_h + start_m * blocksums_stride_m + tl.arange(0, BLOCK_N) * blocksums_stride_n + bsp = ( + blocksums_ptrs + + off_b.to(tl.int64) * blocksums_stride_b + + off_h.to(tl.int64) * blocksums_stride_h + + start_m * blocksums_stride_m + + tl.arange(0, BLOCK_N) * blocksums_stride_n + ) + + # previous m and l values + prev_maxes_final = tl.load(prev_maxes_final_ptrs + softmax_data_offset) + prev_normalization_final = tl.load( + prev_normalization_final_ptrs + softmax_data_offset, mask=(offs_m < seqlen), other=1.0e6 + ) + + # blocksums = tl.zeros([N // BLOCK_N], dtype=tl.float32) + for start_n in range(lo, hi, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load(K_block_ptr) + # q_dot_k = tl.dot(q, k) + q_dot_k = tl.dot(q, k) + # q_dot_k = tl.where(start_n + offs_n[None, :] < 4592, q_dot_k, -1.0e6) + qk = q_dot_k + m_ij = tl.maximum(m_i, tl.max(qk, 1) * qk_scale) + # qk = qk * qk_scale - m_ij[:, None] + qk = qk * qk_scale - m_ij[:, None] + p = tl.math.exp2(qk) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + alpha = tl.math.exp2(m_i - m_ij) + l_i = l_i * alpha + l_ij + # -- update output accumulator -- + acc = acc * alpha[:, None] + # update acc + v = tl.load(V_block_ptr) + # v = tl.where(start_n + offs_n[:, None] < 4592, v, 0).to(tl.bfloat16) + p = p.to(tl.bfloat16) + acc = tl.dot(p, v, acc) + m_i = m_ij + + # ---------------- PREVIOUS PHASE OF SOFTMAX ------------------- + qk_prev = q_dot_k * qk_scale - prev_maxes_final[:, None] + p_prev = tl.math.exp2(qk_prev) + p_prev = p_prev / prev_normalization_final[:, None] + # p_prev = tl.where(offs_m[:, None] < seqlen, p_prev, 0) + blocksums = tl.sum(p_prev, 0) + tl.store(bsp, blocksums, mask=(start_n + offs_n) < seqlen) + + # ----------------- UPDATE POINTERS ----------------- + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + bsp += BLOCK_N * blocksums_stride_n + + return acc, l_i, m_i + + +@triton.autotune(list(filter(keep, configs)), key=["N_CTX", "HEAD_DIM"]) +@triton.jit +def _full_attn_fwd( + Q, + K, + V, + sm_scale, + M, + L, + Out, + seqlen, # + prev_maxes_ptr, # + prev_normalization_final_ptrs, # + blocksums_ptrs, # + softmax_stride_b, + softmax_stride_h, + softmax_stride_n, # + blocksums_stride_b, + blocksums_stride_h, + blocksums_stride_m, + blocksums_stride_n, # + stride_qz, + stride_qh, + stride_qm, + stride_qk, # + stride_kz, + stride_kh, + stride_kn, + stride_kk, # + stride_vz, + stride_vh, + stride_vk, + stride_vn, # + stride_oz, + stride_oh, + stride_om, + stride_on, # + Z, + H, + N_CTX, # + HEAD_DIM: tl.constexpr, # + BLOCK_M: tl.constexpr, # + BLOCK_N: tl.constexpr, # + STAGE: tl.constexpr, # +): + tl.static_assert(BLOCK_N <= HEAD_DIM) + start_m = tl.program_id(0) + off_hz = tl.program_id(1) + off_z = off_hz // H + off_h = off_hz % H + qvk_offset = off_z.to(tl.int64) * stride_qz + off_h.to(tl.int64) * stride_qh + + # block pointers + Q_block_ptr = tl.make_block_ptr( + base=Q + qvk_offset, + shape=(N_CTX, HEAD_DIM), + strides=(stride_qm, stride_qk), + offsets=(start_m * BLOCK_M, 0), + block_shape=(BLOCK_M, HEAD_DIM), + order=(1, 0), + ) + v_order: tl.constexpr = (0, 1) if V.dtype.element_ty == tl.float8e5 else (1, 0) + V_block_ptr = tl.make_block_ptr( + base=V + qvk_offset, + shape=(N_CTX, HEAD_DIM), + strides=(stride_vk, stride_vn), + offsets=(0, 0), + block_shape=(BLOCK_N, HEAD_DIM), + order=v_order, + ) + K_block_ptr = tl.make_block_ptr( + base=K + qvk_offset, + shape=(HEAD_DIM, N_CTX), + strides=(stride_kk, stride_kn), + offsets=(0, 0), + block_shape=(HEAD_DIM, BLOCK_N), + order=(0, 1), + ) + # O_block_ptr = tl.make_block_ptr( + # base=Out + qvk_offset, + # shape=(N_CTX, HEAD_DIM), + # strides=(stride_om, stride_on), + # offsets=(start_m * BLOCK_M, 0), + # block_shape=(BLOCK_M, HEAD_DIM), + # order=(1, 0), + # ) + offs_o = (start_m * BLOCK_M + tl.arange(0, BLOCK_M)[:, None]) * stride_om + tl.arange(0, HEAD_DIM)[ + None, : + ] * stride_on + O_ptrs = Out + qvk_offset + offs_o + + # initialize offsets + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + 1.0 + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + # load scales + qk_scale = sm_scale + qk_scale *= 1.44269504 # 1/log(2) + # load q: it will stay in SRAM throughout + q = tl.load(Q_block_ptr) + # stage 1: off-band + # For causal = True, STAGE = 3 and _attn_fwd_inner gets 1 as its STAGE + # For causal = False, STAGE = 1, and _attn_fwd_inner gets 3 as its STAGE + acc, l_i, m_i = _full_attn_fwd_inner( + acc, + l_i, + m_i, + q, + prev_maxes_ptr, + prev_normalization_final_ptrs, + blocksums_ptrs, + softmax_stride_b, + softmax_stride_h, + softmax_stride_n, + blocksums_stride_b, + blocksums_stride_h, + blocksums_stride_m, + blocksums_stride_n, + K_block_ptr, + V_block_ptr, # + start_m, + qk_scale, + seqlen, # + H, # + BLOCK_M, + HEAD_DIM, + BLOCK_N, # + 4 - STAGE, + offs_m, + offs_n, + N_CTX, + V.dtype.element_ty == tl.float8e5, # + ) + # epilogue + # m_i += tl.math.log2(l_i) + acc = acc / l_i[:, None] + m_ptrs = M + off_hz * N_CTX + offs_m + l_ptrs = L + off_hz * N_CTX + offs_m + tl.store(m_ptrs, m_i, mask=offs_m < seqlen) + tl.store(l_ptrs, l_i, mask=offs_m < seqlen) + # tl.store(O_block_ptr, acc.to(Out.type.element_ty)) + tl.store(O_ptrs, acc.to(Out.type.element_ty), mask=offs_m[:, None] < seqlen) + # tl.store(O_ptrs, acc.to(Out.type.element_ty)) + + +class _full_attention(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, v, sm_scale, prev_maxes, prev_normalization): + # shape constraints + HEAD_DIM_Q, HEAD_DIM_K = q.shape[-1], k.shape[-1] + # when v is in float8_e5m2 it is transposed. + HEAD_DIM_V = v.shape[-1] + assert HEAD_DIM_Q == HEAD_DIM_K and HEAD_DIM_K == HEAD_DIM_V + assert HEAD_DIM_K in {16, 32, 64, 128, 256} + stage = 1 + + # mb = q.shape[2] // 128 if q.shape[2] % 128 == 0 else q.shape[2] // 128 + 1 + mb = triton.cdiv(q.shape[2], 64) + # print(f'mb: {mb}') + # print(f'q.shape[2] // 128: {q.shape[2] // 128}') + # print(f'triton cdiv: {triton.cdiv(q.shape[2], 128)}') + + grid = lambda args: (triton.cdiv(q.shape[2], args["BLOCK_M"]), q.shape[0] * q.shape[1], 1) + # grid = lambda args: (mb, q.shape[0] * q.shape[1], 1) + + # print(f'grid: {grid({})}') + + M = torch.zeros((q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32) + L = torch.zeros_like(M, dtype=torch.float32) + o = torch.empty_like(q) + + blocksums = torch.zeros((q.shape[0], q.shape[1], mb, q.shape[2]), device=q.device, dtype=torch.float32) + # print(f'blocksums: {blocksums.shape}') + seqlen = q.shape[2] + # seqlen = 4592 + _full_attn_fwd[grid]( + q, + k, + v, + sm_scale, + M, + L, + o, + seqlen, # + prev_maxes, # + prev_normalization, # + blocksums, + prev_maxes.stride(0), + prev_maxes.stride(1), + prev_maxes.stride(2), # + blocksums.stride(0), + blocksums.stride(1), + blocksums.stride(2), + blocksums.stride(3), # + q.stride(0), + q.stride(1), + q.stride(2), + q.stride(3), # + k.stride(0), + k.stride(1), + k.stride(2), + k.stride(3), # + v.stride(0), + v.stride(1), + v.stride(2), + v.stride(3), # + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), # + q.shape[0], + q.shape[1], # + N_CTX=q.shape[2], # + HEAD_DIM=HEAD_DIM_K, # + STAGE=stage, + ) + + return o, M, L, blocksums + + +full_attention = _full_attention.apply + + +def partial_col_sum(tensor: torch.Tensor, group_size: int) -> torch.Tensor: + # assert tensor.shape[2] % group_size == 0, "The number of rows must be divisible by group_size." + padding = group_size - tensor.shape[2] % group_size if tensor.shape[2] % group_size != 0 else 0 + tp = F.pad(tensor, (0, 0, 0, padding)) + # print(f'tp: {tp}') + # print(f'tp.shape: {tp.shape}') + # raise ValueError('stop') + return tp.view(tp.shape[0], tp.shape[1], tp.shape[2] // group_size, group_size, tp.shape[3]).sum(dim=3) + + +def reference_impl(): + p = torch.matmul(q, k.transpose(2, 3)) * sm_scale + p_prev_exp = torch.exp2(p.float() * 1.44269504 - prev_m.unsqueeze(-1)) / prev_l.unsqueeze(-1) + ref_blocksums = partial_col_sum(p_prev_exp, 64) + + ref_m = torch.max(p.float(), dim=-1).values * 1.44269504 + p_softmax = torch.softmax(p.float(), dim=-1).to(q.dtype) + ref_out = torch.matmul(p_softmax, v) + l = torch.exp(p.float() - ref_m.unsqueeze(-1)).sum(dim=-1, keepdim=True) + return ref_out, ref_m, l, ref_blocksums + + +baseline_results = {} + + +@triton.testing.perf_report( + [ + triton.testing.Benchmark( + args={}, + x_names=["sparsity_amt"], + x_vals=[x / 10.0 for x in list(range(0, 10, 1))], + styles=[("gray", "dashed"), ("black", "dashed"), ("green", "-")], + line_arg="provider", + line_vals=["y=1-x", "flash", "blocksum"], + line_names=["y=1-x", "FlashAttention", "BlocksumAttention"], + xlabel="Sparsity amount", + ylabel="Kernel Duration (% of FlashAttention)", + plot_name="FlashAttention Performance", + ) + ] +) +def benchmark_blocksum_attn(sparsity_amt, provider): + from torch.nn import functional as F + + global baseline_results + b, h, n, d = 1, 24, 4592, 128 + key = f"{b}_{h}_{n}_{d}" + + q = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + k = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + v = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + o_accum = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + prev_maxes = torch.randn((b, h, n), device="cuda", dtype=torch.float32) + prev_normalization = torch.randn((b, h, n), device="cuda", dtype=torch.float32) + + quantiles = [0.5, 0.2, 0.8] + rescale = lambda tuple: ( + tuple[0] / baseline_results[key][0], + tuple[1] / baseline_results[key][1], + tuple[2] / baseline_results[key][2], + ) + # rescale = lambda tuple: tuple + + if provider == "flash": + if key not in baseline_results: + # with sdpa_kernel(SDPBackend.FLASH_ATTENTION): + # ans = triton.testing.do_bench(lambda: F.scaled_dot_product_attention(q, k, v, is_causal=False), quantiles=quantiles) + ans = triton.testing.do_bench( + lambda: F.scaled_dot_product_attention(q, k, v, is_causal=False), quantiles=quantiles + ) + baseline_results[key] = ans + return rescale(baseline_results[key]) + elif provider == "blocksum": + results = triton.testing.do_bench( + lambda: full_attention(q, k, v, 0.5, prev_maxes, prev_normalization), + quantiles=quantiles, + warmup=100, + rep=1000, + ) + minms, meanms, maxms = results + # TFLOPs calculation + flops = 4 * b * h * n * n * d + print(f"tflops: {flops * 1e-12 / (meanms * 1e-3)}") + return rescale(results) + elif provider == "y=1-x": + return (1 - sparsity_amt, 1 - sparsity_amt, 1 - sparsity_amt) + else: + raise ValueError(f"Invalid provider: {provider}") + + +if __name__ == "__main__": + # Z, H, N_CTX, HEAD_DIM = 1, 2, 1024, 64 + Z, H, N_CTX, HEAD_DIM = 1, 24, 34 * 128 + 0, 128 + BLOCK_M = 64 + dtype = torch.bfloat16 + torch.manual_seed(20) + q = ( + torch.empty((Z, H, N_CTX, HEAD_DIM), dtype=dtype, device=DEVICE) + .normal_(mean=0.0, std=0.5) + .requires_grad_(True) + ) + k = ( + torch.empty((Z, H, N_CTX, HEAD_DIM), dtype=dtype, device=DEVICE) + .normal_(mean=0.0, std=0.5) + .requires_grad_(True) + ) + v = ( + torch.empty((Z, H, N_CTX, HEAD_DIM), dtype=dtype, device=DEVICE) + .normal_(mean=0.0, std=0.5) + .requires_grad_(True) + ) + prev_m = ( + torch.empty((Z, H, N_CTX), dtype=torch.float32, device=DEVICE).normal_(mean=0.0, std=0.5).requires_grad_(False) + ) + prev_l = ( + torch.empty((Z, H, N_CTX), dtype=torch.float32, device=DEVICE) + .normal_(mean=0.0, std=0.5) + .abs() + .requires_grad_(False) + ) + sm_scale = 1 / HEAD_DIM**0.5 + + ref_out, ref_m, ref_l, ref_blocksums = reference_impl() + # prev_m = ref_m + # prev_l = ref_l.squeeze(-1) + # print(f'prev_l: {prev_l[0, -1, -128:]}') + print(f"prev_m shape: {prev_m.shape}") + print(f"prev_l shape: {prev_l.shape}") + tri_out, tri_m, tri_l, tri_blocksums = full_attention(q, k, v, sm_scale, prev_m, prev_l) + print("Output difference (max):", (ref_out - tri_out).abs().max().item()) + assert (ref_out - tri_out).abs().max().item() < 0.01, ( + "The output of the reference attention implementation and the tri implementation are not close enough." + ) + print("Max m difference:", (tri_m - ref_m).abs().max().item()) + assert (tri_m - ref_m).abs().max().item() < 0.02, ( + "The m values of the reference attention implementation and the tri implementation are not close enough." + ) + print("Blocksums difference (mean):", (tri_blocksums - ref_blocksums).abs().mean().item()) + # breakpoint() + print(f"tri_blocksums: {tri_blocksums[0, -1, -1]}") + print(f"ref_blocksums: {ref_blocksums[0, -1, -1]}") + # breakpoint() + assert (tri_blocksums - ref_blocksums).abs().mean().item() < 0.7, ( + "The blocksums of the reference attention implementation and the tri implementation are not close enough." + ) + print("✅ All correctness checks passed.") + + benchmark_blocksum_attn.run(show_plots=False, print_data=True) diff --git a/nemo_vfm/sparse_attention/kernels/tl_sparse_attention.py b/nemo_vfm/sparse_attention/kernels/tl_sparse_attention.py new file mode 100644 index 00000000..423311bb --- /dev/null +++ b/nemo_vfm/sparse_attention/kernels/tl_sparse_attention.py @@ -0,0 +1,477 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import math + +import torch +import triton +import triton.language as tl +from einops import rearrange + + +# from tl_base_attention import attention as tl_dense + +DEVICE = "cuda" + +cdiv = lambda a, b: (a + b - 1) // b +configs = [ + triton.Config({"BLOCK_M": BM, "BLOCK_N": BN}, num_stages=s, num_warps=w) + for BM in [64] + for BN in [64] + for s in [3, 4, 7] + for w in [4, 8] +] + + +def keep(conf): + BLOCK_M = conf.kwargs["BLOCK_M"] + BLOCK_N = conf.kwargs["BLOCK_N"] + if BLOCK_M * BLOCK_N < 128 * 128 and conf.num_warps == 8: + return False + return True + + +@triton.jit +def _sparse_attn_fwd_inner( + acc, + l_i, + m_i, + q, # + K_block_ptr_orig, + V_block_ptr_orig, # + start_m, + qk_scale, # + BLOCK_M: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_N: tl.constexpr, # + STAGE: tl.constexpr, + offs_m: tl.constexpr, + offs_n: tl.constexpr, # + N_CTX: tl.constexpr, + fp8_v: tl.constexpr, # + stride_k_seqlen, + stride_v_seqlen, # + sparsity_indices_ptr, + sparsity_counts_ptr, # +): + sparsity_count = tl.load(sparsity_counts_ptr + start_m) + sparsity_offsets = tl.arange(0, BLOCK_N) + sparsity_indices_ptr += start_m * N_CTX + sparsity_offsets + # sparsity_indices = tl.load(sparsity_indices_ptr) + n_iters = tl.cdiv(sparsity_count, BLOCK_N) + cur_iter = 0 + # loop over k, v and update accumulator + for start_n in range(0, sparsity_count, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + sparsity_indices = tl.load(sparsity_indices_ptr) + K_block_ptr = K_block_ptr_orig + (sparsity_indices[None, :]) * stride_k_seqlen + V_block_ptr = V_block_ptr_orig + (sparsity_indices[:, None]) * stride_v_seqlen + # Commented out lines are for when we use random sparsity counts, in production it's always a multiple of BLOCK_N = 64 + # K_block_ptr = K_block_ptr_orig + (sparsity_indices[None, :] % N_CTX) * stride_k_seqlen + # V_block_ptr = V_block_ptr_orig + (sparsity_indices[:, None] % N_CTX) * stride_v_seqlen + # is_valid_mask = sparsity_offsets < sparsity_count - start_n # shape (BLOCK_N,) + k = tl.load(K_block_ptr) + qk = tl.dot(q, k) + m_ij = tl.maximum(m_i, tl.max(qk, 1) * qk_scale) + # qk = qk * qk_scale + tl.where(is_valid_mask[None, :], 0, -1.0e6) + # qk -= m_ij[:, None] + qk = qk * qk_scale - m_ij[:, None] # use fused multiply add! + p = tl.math.exp2(qk) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + alpha = tl.math.exp2(m_i - m_ij) + l_i = l_i * alpha + l_ij + # -- update output accumulator -- + acc = acc * alpha[:, None] + # update acc + v = tl.load(V_block_ptr) + p = p.to(tl.bfloat16) + acc = tl.dot(p, v, acc) + m_i = m_ij + sparsity_indices_ptr += BLOCK_N + cur_iter += 1 + + return acc, l_i, m_i + + +@triton.autotune(list(filter(keep, configs)), key=["N_CTX", "HEAD_DIM"]) +@triton.jit +def _sparse_attn_fwd( + Q, + K, + V, + sm_scale, + M, + L, + Out, + Out_accum, + Out_scale: tl.constexpr, # + sparsity_indices, + sparsity_counts, # + stride_qz, + stride_qh, + stride_qm, + stride_qk, # + stride_kz, + stride_kh, + stride_kn, + stride_kk, # + stride_vz, + stride_vh, + stride_vk, + stride_vn, # + stride_oz, + stride_oh, + stride_om, + stride_on, # + stride_spiz, + stride_spih, # + stride_spcz, + stride_spch, # + Z, + H, + N_CTX, # + HEAD_DIM: tl.constexpr, # + BLOCK_M: tl.constexpr, # + BLOCK_N: tl.constexpr, # + STAGE: tl.constexpr, # +): + tl.static_assert(BLOCK_N <= HEAD_DIM) + start_m = tl.program_id(0) + off_hz = tl.program_id(1) + off_z = off_hz // H + off_h = off_hz % H + qvk_offset = off_z.to(tl.int64) * stride_qz + off_h.to(tl.int64) * stride_qh + + spi_offset = off_z.to(tl.int64) * stride_spiz + off_h.to(tl.int64) * stride_spih + spi_ptr = sparsity_indices + spi_offset + spc_offset = off_z.to(tl.int64) * stride_spcz + off_h.to(tl.int64) * stride_spch + spc_ptr = sparsity_counts + spc_offset + + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + offs_headsize = tl.arange(0, HEAD_DIM) + + # block pointers + Q_block_ptr = Q + qvk_offset + offs_m[:, None] * stride_qm + offs_headsize[None, :] * stride_qk + K_block_ptr = K + qvk_offset + (offs_n[None, :] // BLOCK_N) * stride_kn + offs_headsize[:, None] * stride_kk + V_block_ptr = V + qvk_offset + (offs_n[:, None] // BLOCK_N) * stride_vk + offs_headsize[None, :] * stride_vn + O_block_ptr = Out + qvk_offset + offs_m[:, None] * stride_om + offs_headsize[None, :] * stride_on + O_accum_block_ptr = Out_accum + qvk_offset + offs_m[:, None] * stride_om + offs_headsize[None, :] * stride_on + # initialize offsets + # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + 1.0 + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + # load scales + qk_scale = sm_scale + qk_scale *= 1.44269504 # 1/log(2) + # load q: it will stay in SRAM throughout + # qo_mask = (offs_m < N_CTX)[:, None] + q = tl.load(Q_block_ptr) + # stage 1: off-band + # For causal = True, STAGE = 3 and _sparse_attn_fwd_inner gets 1 as its STAGE + # For causal = False, STAGE = 1, and _sparse_attn_fwd_inner gets 3 as its STAGE + acc, l_i, m_i = _sparse_attn_fwd_inner( + acc, + l_i, + m_i, + q, + K_block_ptr, + V_block_ptr, # + start_m, + qk_scale, # + BLOCK_M, + HEAD_DIM, + BLOCK_N, # + 4 - STAGE, + offs_m, + offs_n, + N_CTX, + V.dtype.element_ty == tl.float8e5, # + stride_kn, + stride_vk, # + spi_ptr, + spc_ptr, # + ) + # epilogue + # m_i += tl.math.log2(l_i) + acc = acc / l_i[:, None] + m_ptrs = M + off_hz * N_CTX + offs_m + l_ptrs = L + off_hz * N_CTX + offs_m + tl.store(m_ptrs, m_i) + tl.store(l_ptrs, l_i) + acc *= Out_scale # will get optimized out when Out_scale is 1.0 since it's tl.constexpr + acc += tl.load(O_accum_block_ptr) + tl.store(O_block_ptr, acc.to(Out.type.element_ty)) + + +class _sparse_attention(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, v, o_accum, sm_scale, sparsity_indices, sparsity_counts, O_scale=1.0): + # shape constraints + HEAD_DIM_Q, HEAD_DIM_K = q.shape[-1], k.shape[-1] + # when v is in float8_e5m2 it is transposed. + HEAD_DIM_V = v.shape[-1] + assert HEAD_DIM_Q == HEAD_DIM_K and HEAD_DIM_K == HEAD_DIM_V + assert HEAD_DIM_K in {16, 32, 64, 128, 256} + o = torch.empty_like(q) + stage = 1 + extra_kern_args = {} + + grid = lambda args: (triton.cdiv(q.shape[2], args["BLOCK_M"]), q.shape[0] * q.shape[1], 1) + M = torch.empty((q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32) + L = torch.empty((q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32) + _sparse_attn_fwd[grid]( + q, + k, + v, + sm_scale, + M, + L, + o, + o_accum, + O_scale, # + sparsity_indices, + sparsity_counts, # + q.stride(0), + q.stride(1), + q.stride(2), + q.stride(3), # + k.stride(0), + k.stride(1), + k.stride(2), + k.stride(3), # + v.stride(0), + v.stride(1), + v.stride(2), + v.stride(3), # + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), # + sparsity_indices.stride(0), + sparsity_indices.stride(1), # + sparsity_counts.stride(0), + sparsity_counts.stride(1), # + q.shape[0], + q.shape[1], # + N_CTX=q.shape[2], # + HEAD_DIM=HEAD_DIM_K, # + STAGE=stage, # + **extra_kern_args, + ) + + return o, M, L + + +sparse_attention = _sparse_attention.apply + + +def get_sparsity_data(b, h, seqlen, sparsity_amount=0.5): + import random + + random.seed(0) + + assert sparsity_amount < 1, "sparsity_amount must be less than 1" + BLOCK_SIZE_M = 64 + BLOCK_SIZE_N = 64 + num_m_blocks = cdiv(seqlen, BLOCK_SIZE_M) + + dtype = torch.int32 + device = "cuda" + sparsity_indices = torch.full((b, h, num_m_blocks, seqlen), fill_value=0, device=device, dtype=dtype) + sparsity_indices_counts = torch.full((b, h, num_m_blocks), fill_value=0, device=device, dtype=dtype) + blockmask = torch.zeros((b, h, num_m_blocks, seqlen), device=device, dtype=torch.bool) + + for i in range(b): + for j in range(h): + for k in range(num_m_blocks): + # make it a multiple of BLOCK_SIZE_N since this is a constraint of the kernel + sparsity_count = int(seqlen * (1 - sparsity_amount) / BLOCK_SIZE_N) * BLOCK_SIZE_N + indices = torch.randperm(seqlen, device=device, dtype=dtype)[:sparsity_count] + # print(f'len(indices): {len(indices)}') + sparsity_indices[i, j, k, : len(indices)] = indices + sparsity_indices_counts[i, j, k] = sparsity_count + blockmask[i, j, k, indices] = True + + blockmask = rearrange(blockmask.unsqueeze(3).expand(-1, -1, -1, BLOCK_SIZE_M, -1), "b h mb bm n -> b h (mb bm) n")[ + :, :, :seqlen + ] + + # # add a few extra values to make sure that our masking works properly + # sparsity_difference_range = 15 + # for i in range(num_m_blocks): + # offset = random.randint(-sparsity_difference_range, 0) + # num_indices = min(max(int(seqlen * (1 - sparsity_amount)), 1) - 0, seqlen) + # indices = list(range(0, seqlen)) + # random.shuffle(indices) + # indices = indices[:num_indices] + # sparsity_indices[i, :num_indices] = torch.tensor(indices, device=device, dtype=dtype) + # sparsity_indices_counts[i] = num_indices + + assert sparsity_indices.shape[-1] == seqlen, "stride of sparsity_indices must be seqlen" + # print(f'sparsity_indices: {sparsity_indices}') + # print(f'sparsity_indices_counts: {sparsity_indices_counts}') + # print(f'sparsity_indices shape: {sparsity_indices.shape}') + # print(f'sparsity_indices_counts shape: {sparsity_indices_counts.shape}') + # print(f'blockmask shape: {blockmask.shape}') + # raise Exception('stop') + return sparsity_indices, sparsity_indices_counts, blockmask + + +def test_sparse_op(b, h, n, d, sp_inds, sp_counts, blockmask, dtype=torch.bfloat16): + torch.manual_seed(20) + q = torch.randn((b, h, n, d), device="cuda", dtype=dtype) + k = torch.randn((b, h, n, d), device="cuda", dtype=dtype) + v = torch.randn((b, h, n, d), device="cuda", dtype=dtype) + o_accum = torch.randn((b, h, n, d), device="cuda", dtype=dtype) + sparsity_amount = 0.5 + kt = k.transpose(2, 3).contiguous() + sm_scale = 1 / math.sqrt(d) + + # qkt_rows = [] + # for i in range(sp_counts.shape[0]): + # sp_count = sp_counts[0,0, i].item() + # sp_inds_i = sp_inds[0,0, i, :sp_count] + # sparsity_mask = torch.zeros((d, n), device='cuda', dtype=torch.bool) + # sparsity_mask[:, sp_inds_i] = True + # kt_block_sparse = torch.where(sparsity_mask, kt, 0) + # q_block_dense = q[:, :, i * 64:(i + 1) * 64, :] + # scaled = (q_block_dense @ kt_block_sparse) * sm_scale + # scaled.masked_fill_(~sparsity_mask[0].unsqueeze(0), float("-inf")) + # qkt_rows.append(scaled) + + # qkt = torch.cat(qkt_rows, dim=2) + # breakpoint() + # sparse_attn_ref = torch.softmax(qkt, dim=-1) @ v + o_accum + + logits = torch.matmul(q, kt) * sm_scale + torch.where(blockmask, 0, float("-inf")) + sparse_attn_ref = torch.softmax(logits.float(), dim=-1).to(dtype) @ v + o_accum + + print(f"sparse ref shape: {sparse_attn_ref.shape}") + tri_out, M, L = sparse_attention(q, k, v, o_accum, sm_scale, sp_inds, sp_counts) + print(f"sparse tri shape: {tri_out.shape}") + return sparse_attn_ref, tri_out + + +gpu_name = torch.cuda.get_device_name(0).replace("NVIDIA", "").strip() + +baseline_results = {} + + +@triton.testing.perf_report( + [ + triton.testing.Benchmark( + args={}, + x_names=["sparsity_amt"], + # x_vals=[x/10.0 for x in list(range(0, 10, 1))], + x_vals=[0.835], + styles=[("gray", "dashed"), ("black", "dashed"), ("green", "-"), ("blue", "-")], + line_arg="provider", + line_vals=["y=1-x", "flash", "tl-dense", "sparse"], + line_names=["y=1-x", "FlashAttention", "TL Dense", "SparseAttention"], + xlabel="Sparsity amount", + ylabel=f"Kernel Duration (% of FlashAttention) {gpu_name}", + plot_name=f"FlashAttention Performance {gpu_name}", + ) + ] +) +def benchmark_sparse_attn(sparsity_amt, provider): + from torch.nn import functional as F + + global baseline_results + b, h, n, d = 1, 24, 4352, 128 + sp_ind, sp_counts, blockmask = get_sparsity_data(b, h, n, sparsity_amt) + key = f"{b}_{h}_{n}_{d}_{gpu_name}" + + q = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + k = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + v = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + o_accum = torch.randn((b, h, n, d), device="cuda", dtype=torch.bfloat16) + # q = torch.load(f'../sp-attn-q.pt') + # k = torch.load(f'../sp-attn-k.pt') + # v = torch.load(f'../sp-attn-v.pt') + # o_accum = torch.load(f'../sp-attn-oc.pt') + # sp_ind = torch.load(f'../sp-attn-inds.pt') + # sp_counts = torch.load(f'../sp-attn-counts.pt') + + quantiles = [0.5, 0.2, 0.8] + rescale = lambda tuple: ( + tuple[0] / baseline_results[key][0], + tuple[1] / baseline_results[key][1], + tuple[2] / baseline_results[key][2], + ) + # rescale = lambda tuple: tuple + + if provider == "flash": + if key not in baseline_results: + # with sdpa_kernel(SDPBackend.FLASH_ATTENTION): + # ans = triton.testing.do_bench(lambda: F.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False), quantiles=quantiles) + ans = triton.testing.do_bench( + lambda: F.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False), + quantiles=quantiles, + ) + baseline_results[key] = ans + flops = 4 * b * h * n * n * d + print(f"tflops fa2: {flops * 1e-12 / (ans[1] * 1e-3)}") + # return rescale(baseline_results[key]) + return baseline_results[key] + elif provider == "tl-dense": + results = triton.testing.do_bench( + lambda: tl_dense(q, k, v, False, 0.5), quantiles=quantiles, warmup=100, rep=1000 + ) + minms, meanms, maxms = results + # TFLOPs calculation + flops = 4 * b * h * n * n * d + print(f"tflops tl-dense: {flops * 1e-12 / (meanms * 1e-3)}") + # return rescale(results) + return results + elif provider == "sparse": + results = triton.testing.do_bench( + lambda: sparse_attention(q, k, v, o_accum, 0.5, sp_ind, sp_counts), + quantiles=quantiles, + warmup=100, + rep=1000, + ) + minms, meanms, maxms = results + # TFLOPs calculation + flops = 4 * b * h * n * n * d * (1 - sparsity_amt) + print(f"tflops sparse: {flops * 1e-12 / (meanms * 1e-3)}") + # return rescale(results) + return results + elif provider == "y=1-x": + return (1 - sparsity_amt, 1 - sparsity_amt, 1 - sparsity_amt) + else: + raise ValueError(f"Invalid provider: {provider}") + + +if __name__ == "__main__": + b, h, n, d = 1, 24, 34 * 128 + 0, 128 + print(f"b: {b}, h: {h}, n: {n}, d: {d}") + sparsity_amount = 0.5 + sp_inds, sp_counts, blockmask = get_sparsity_data(b, h, n, sparsity_amount) + ref_out, tri_out = test_sparse_op(b, h, n, d, sp_inds, sp_counts, blockmask) + print(f"ref_out: {ref_out[0, -1, :10, :10]}") + print(f"tri_out: {tri_out[0, -1, :10, :10]}") + print(f"distance: {torch.dist(ref_out, tri_out)}") + print(f"max diff: {torch.max(torch.abs(ref_out - tri_out))}") + if not torch.allclose(ref_out, tri_out, atol=1e-2): + print("❌ Sparse attention output does not match reference output") + else: + print("✅ All correctness checks passed.") + + benchmark_sparse_attn.run(show_plots=True, print_data=True) diff --git a/nemo_vfm/sparse_attention/sparse_attn.py b/nemo_vfm/sparse_attention/sparse_attn.py new file mode 100644 index 00000000..d0a443b8 --- /dev/null +++ b/nemo_vfm/sparse_attention/sparse_attn.py @@ -0,0 +1,122 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=C0115,C0116,C0301 + +import flash_attn.flash_attn_interface +import torch.nn.functional as F + + +original_flash_attn_varlen_forward = flash_attn.flash_attn_interface._flash_attn_varlen_forward +original_flash_attn_forward = flash_attn.flash_attn_interface._flash_attn_forward +original_sdpa = F.scaled_dot_product_attention + + +def flash_attn_func(q, k, v, *args, **kwargs): + if q.shape != k.shape or q.shape != v.shape: + return original_flash_attn_forward(q, k, v, *args, **kwargs) + kwargs["should_rearrange"] = True + o = invoke_sparse_layer(q, k, v, *args, **kwargs) + if o is None: + return original_flash_attn_forward(q, k, v, *args, **kwargs) + return (o, None, None, None, o, q[:, :, :, 0], None, None) + + +def flash_attn_varlen_func(q, k, v, *args, **kwargs): + if q.shape != k.shape or q.shape != v.shape: + return original_flash_attn_varlen_forward(q, k, v, *args, **kwargs) + kwargs["should_rearrange"] = True + o = invoke_sparse_layer(q, k, v, *args, should_rearrange=True, **kwargs) + if o is None: + return original_flash_attn_varlen_forward(q, k, v, *args, **kwargs) + return (o, None, None, None, o, q[:, :, :, 0], None, None) + + +def sdpa_func(q, k, v, *args, **kwargs): + if q.shape != k.shape or q.shape != v.shape: + return original_sdpa(q, k, v, *args, **kwargs) + o = invoke_sparse_layer(q, k, v, *args, should_rearrange=False, **kwargs) + if o is None: + return original_sdpa(q, k, v, *args, **kwargs) + return o + + +flash_attn.flash_attn_interface._flash_attn_varlen_forward = flash_attn_varlen_func +flash_attn.flash_attn_interface._flash_attn_forward = flash_attn_func +F.scaled_dot_product_attention = sdpa_func + +from chipmunk.modules import SparseDiffAttn +from chipmunk.util import GLOBAL_CONFIG, LayerCounter +from einops import rearrange + + +_, layer_counter = LayerCounter.build_for_layer(is_mlp_sparse=False, is_attn_sparse=True) +sparse_attn_layers = None +num_heads = None +tensor_parallel_size = None + + +def setup_sparse_attn(file_path): + import chipmunk.util + + chipmunk.util.config.load_from_file(file_path) + global sparse_attn_layers + global num_heads + global tensor_parallel_size + + layer_counter.num_layers = GLOBAL_CONFIG["model_config"]["num_layers"] + layer_counter.num_submodules_per_layer = GLOBAL_CONFIG["model_config"]["context_parallel_size"] + tensor_parallel_size = GLOBAL_CONFIG["model_config"]["tensor_parallel_size"] + + GLOBAL_CONFIG["num_steps"] = GLOBAL_CONFIG["model_config"]["num_steps"] + num_heads = GLOBAL_CONFIG["model_config"]["num_heads"] + sparse_attn_layers = [ + [SparseDiffAttn(i, layer_counter) for j in range(layer_counter.num_submodules_per_layer)] + for i in range(layer_counter.num_layers) + ] + + +rearrange_forward = lambda o: rearrange(o, "b n h d -> b h n d").contiguous() +rearrange_reverse = lambda o: rearrange(o, "b h n d -> b n h d").contiguous() +has_initialized = False + + +def invoke_sparse_layer(q, k, v, *args, should_rearrange, **kwargs): + if should_rearrange: + q, k, v = rearrange_forward(q), rearrange_forward(k), rearrange_forward(v) + w, h, d = ( + GLOBAL_CONFIG["model_config"]["latent_vector_shape"]["width"], + GLOBAL_CONFIG["model_config"]["latent_vector_shape"]["height"], + GLOBAL_CONFIG["model_config"]["latent_vector_shape"]["depth"], + ) + expected_sequence_length = w * h * d + print(f"expected_sequence_length: {expected_sequence_length}, q.shape: {q.shape}") + if q.shape[2] < expected_sequence_length: + # Don't process sparse attention for short sequences - this means that we're running a second model in the background that we don't care about + return None + print("Sparse attention", q.shape, k.shape, v.shape) + global has_initialized + if not has_initialized: + sparse_attn_layers[0][0].initialize_static_mask( + seq_shape=(w, h, d), txt_len=0, local_heads_num=num_heads // tensor_parallel_size, device="cuda" + ) + has_initialized = True + + cur_state = layer_counter.cur_inference_step, layer_counter.cur_layer, layer_counter.cur_layer_submodule + step, layer_idx, submodule_idx = cur_state + o = sparse_attn_layers[layer_idx][submodule_idx](q, k, v) + + if should_rearrange: + o = rearrange_reverse(o).contiguous() + return o diff --git a/nemo_vfm/sparse_attention/training.md b/nemo_vfm/sparse_attention/training.md new file mode 100644 index 00000000..a27f0ffc --- /dev/null +++ b/nemo_vfm/sparse_attention/training.md @@ -0,0 +1,17 @@ +Clone the repository + +```bash +git clone https://gitlab-master.nvidia.com/sgovande/Cosmos/ Cosmos-gitlab +git clone https://gitlab-master.nvidia.com/dl/nemo/nemo-vfm nemo-vfm +cd nemo-vfm && git checkout training-sparse-attn && cd .. +``` + +Modify `Cosmos-gitlab/cosmos1/models/diffusion/nemo/post_training/multinode.sh` with the following: +1. Update paths to `nemo-vfm` +2. Update the environment variable `MOUNT` to point to your new location + +```bash +cd Cosmos-gitlab +cd cosmos1/models/diffusion/nemo/post_training +sbatch multinode.sh +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 758d1678..20a4d190 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,19 +157,19 @@ extend-exclude = [ # Configure imports sorting to match isort select = [ "F541", # f-string without any placeholders - "F841", # local variable assigned but never used "F401", # imported but unused - "E741", # ambiguous variable name - "F821", # undefined name "E266", # too many leading '#' for block comment "I", # isort - "D101", # docstring - "D103", ] # Additional rules can be added here ignore = [ + "F821", # undefined name + "E741", # ambiguous variable name + "F841", # local variable assigned but never used "E501", # Line too long - handled by formatter + "D101", # docstring + "D103", # Missing docstring in public function ] [tool.ruff.lint.isort] known-first-party = ["nemo_vfm"]