update: Add necessary file, remove unused imports, delete unused code…

README.md

@@ -10,6 +10,13 @@ We conduct a comprehensive analysis for the problem of existing spiking neuron m
 Based on this, we propose a bounded spiking neuron to build the discontinuous density field.
 ## Usage
 
+### Download Data
+
+Download data for two example datasets: `lego` and `fern`
+```
+bash download_example_data.sh
+```
+
 #### Data Convention
 
 The Blender data is organized as follows:
@@ -46,5 +53,7 @@ pip install -r requirements.txt
 - **Training Blender**
 
 ```shell 
-python nerf_vth2.py --config ./config/xx.txt 
+python nerf_vth2.py --config ./configs/{DATASET}.txt 
 ```
+
+replace `{DATASET}` with `trex` | `horns` | `flower` | `fortress` | `lego` | etc.

download_example_data.sh

@@ -0,0 +1,6 @@
+wget http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/tiny_nerf_data.npz
+mkdir -p data
+cd data
+wget http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/nerf_example_data.zip
+unzip nerf_example_data.zip
+cd ..

load_LINEMOD.py

@@ -0,0 +1,95 @@
+import os
+import torch
+import numpy as np
+import imageio 
+import json
+import torch.nn.functional as F
+import cv2
+
+
+trans_t = lambda t : torch.Tensor([
+    [1,0,0,0],
+    [0,1,0,0],
+    [0,0,1,t],
+    [0,0,0,1]]).float()
+
+rot_phi = lambda phi : torch.Tensor([
+    [1,0,0,0],
+    [0,np.cos(phi),-np.sin(phi),0],
+    [0,np.sin(phi), np.cos(phi),0],
+    [0,0,0,1]]).float()
+
+rot_theta = lambda th : torch.Tensor([
+    [np.cos(th),0,-np.sin(th),0],
+    [0,1,0,0],
+    [np.sin(th),0, np.cos(th),0],
+    [0,0,0,1]]).float()
+
+
+def pose_spherical(theta, phi, radius):
+    c2w = trans_t(radius)
+    c2w = rot_phi(phi/180.*np.pi) @ c2w
+    c2w = rot_theta(theta/180.*np.pi) @ c2w
+    c2w = torch.Tensor(np.array([[-1,0,0,0],[0,0,1,0],[0,1,0,0],[0,0,0,1]])) @ c2w
+    return c2w
+
+
+def load_LINEMOD_data(basedir, half_res=False, testskip=1):
+    splits = ['train', 'val', 'test']
+    metas = {}
+    for s in splits:
+        with open(os.path.join(basedir, 'transforms_{}.json'.format(s)), 'r') as fp:
+            metas[s] = json.load(fp)
+
+    all_imgs = []
+    all_poses = []
+    counts = [0]
+    for s in splits:
+        meta = metas[s]
+        imgs = []
+        poses = []
+        if s=='train' or testskip==0:
+            skip = 1
+        else:
+            skip = testskip
+
+        for idx_test, frame in enumerate(meta['frames'][::skip]):
+            fname = frame['file_path']
+            if s == 'test':
+                print(f"{idx_test}th test frame: {fname}")
+            imgs.append(imageio.imread(fname))
+            poses.append(np.array(frame['transform_matrix']))
+        imgs = (np.array(imgs) / 255.).astype(np.float32) # keep all 4 channels (RGBA)
+        poses = np.array(poses).astype(np.float32)
+        counts.append(counts[-1] + imgs.shape[0])
+        all_imgs.append(imgs)
+        all_poses.append(poses)
+
+    i_split = [np.arange(counts[i], counts[i+1]) for i in range(3)]
+
+    imgs = np.concatenate(all_imgs, 0)
+    poses = np.concatenate(all_poses, 0)
+
+    H, W = imgs[0].shape[:2]
+    focal = float(meta['frames'][0]['intrinsic_matrix'][0][0])
+    K = meta['frames'][0]['intrinsic_matrix']
+    print(f"Focal: {focal}")
+
+    render_poses = torch.stack([pose_spherical(angle, -30.0, 4.0) for angle in np.linspace(-180,180,40+1)[:-1]], 0)
+
+    if half_res:
+        H = H//2
+        W = W//2
+        focal = focal/2.
+
+        imgs_half_res = np.zeros((imgs.shape[0], H, W, 3))
+        for i, img in enumerate(imgs):
+            imgs_half_res[i] = cv2.resize(img, (W, H), interpolation=cv2.INTER_AREA)
+        imgs = imgs_half_res
+        # imgs = tf.image.resize_area(imgs, [400, 400]).numpy()
+
+    near = np.floor(min(metas['train']['near'], metas['test']['near']))
+    far = np.ceil(max(metas['train']['far'], metas['test']['far']))
+    return imgs, poses, render_poses, [H, W, focal], K, i_split, near, far
+
+

load_deepvoxels.py

@@ -0,0 +1,110 @@
+import os
+import numpy as np
+import imageio 
+
+
+def load_dv_data(scene='cube', basedir='/data/deepvoxels', testskip=8):
+
+
+    def parse_intrinsics(filepath, trgt_sidelength, invert_y=False):
+        # Get camera intrinsics
+        with open(filepath, 'r') as file:
+            f, cx, cy = list(map(float, file.readline().split()))[:3]
+            grid_barycenter = np.array(list(map(float, file.readline().split())))
+            near_plane = float(file.readline())
+            scale = float(file.readline())
+            height, width = map(float, file.readline().split())
+
+            try:
+                world2cam_poses = int(file.readline())
+            except ValueError:
+                world2cam_poses = None
+
+        if world2cam_poses is None:
+            world2cam_poses = False
+
+        world2cam_poses = bool(world2cam_poses)
+
+        print(cx,cy,f,height,width)
+
+        cx = cx / width * trgt_sidelength
+        cy = cy / height * trgt_sidelength
+        f = trgt_sidelength / height * f
+
+        fx = f
+        if invert_y:
+            fy = -f
+        else:
+            fy = f
+
+        # Build the intrinsic matrices
+        full_intrinsic = np.array([[fx, 0., cx, 0.],
+                                   [0., fy, cy, 0],
+                                   [0., 0, 1, 0],
+                                   [0, 0, 0, 1]])
+
+        return full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses
+
+
+    def load_pose(filename):
+        assert os.path.isfile(filename)
+        nums = open(filename).read().split()
+        return np.array([float(x) for x in nums]).reshape([4,4]).astype(np.float32)
+
+
+    H = 512
+    W = 512
+    deepvoxels_base = '{}/train/{}/'.format(basedir, scene)
+
+    full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses = parse_intrinsics(os.path.join(deepvoxels_base, 'intrinsics.txt'), H)
+    print(full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses)
+    focal = full_intrinsic[0,0]
+    print(H, W, focal)
+
+
+    def dir2poses(posedir):
+        poses = np.stack([load_pose(os.path.join(posedir, f)) for f in sorted(os.listdir(posedir)) if f.endswith('txt')], 0)
+        transf = np.array([
+            [1,0,0,0],
+            [0,-1,0,0],
+            [0,0,-1,0],
+            [0,0,0,1.],
+        ])
+        poses = poses @ transf
+        poses = poses[:,:3,:4].astype(np.float32)
+        return poses
+
+    posedir = os.path.join(deepvoxels_base, 'pose')
+    poses = dir2poses(posedir)
+    testposes = dir2poses('{}/test/{}/pose'.format(basedir, scene))
+    testposes = testposes[::testskip]
+    valposes = dir2poses('{}/validation/{}/pose'.format(basedir, scene))
+    valposes = valposes[::testskip]
+
+    imgfiles = [f for f in sorted(os.listdir(os.path.join(deepvoxels_base, 'rgb'))) if f.endswith('png')]
+    imgs = np.stack([imageio.imread(os.path.join(deepvoxels_base, 'rgb', f))/255. for f in imgfiles], 0).astype(np.float32)
+
+
+    testimgd = '{}/test/{}/rgb'.format(basedir, scene)
+    imgfiles = [f for f in sorted(os.listdir(testimgd)) if f.endswith('png')]
+    testimgs = np.stack([imageio.imread(os.path.join(testimgd, f))/255. for f in imgfiles[::testskip]], 0).astype(np.float32)
+
+    valimgd = '{}/validation/{}/rgb'.format(basedir, scene)
+    imgfiles = [f for f in sorted(os.listdir(valimgd)) if f.endswith('png')]
+    valimgs = np.stack([imageio.imread(os.path.join(valimgd, f))/255. for f in imgfiles[::testskip]], 0).astype(np.float32)
+
+    all_imgs = [imgs, valimgs, testimgs]
+    counts = [0] + [x.shape[0] for x in all_imgs]
+    counts = np.cumsum(counts)
+    i_split = [np.arange(counts[i], counts[i+1]) for i in range(3)]
+
+    imgs = np.concatenate(all_imgs, 0)
+    poses = np.concatenate([poses, valposes, testposes], 0)
+
+    render_poses = testposes
+
+    print(poses.shape, imgs.shape)
+
+    return imgs, poses, render_poses, [H,W,focal], i_split
+
+