big errors #2
Description
I had a problem with Caffe installation on ubuntu20.4 and I sent an email to you. You suggested me a code in the Torch framework (https://github.com/KamitaniLab/brain-decoding-cookbook-public/tree/main/reconstruction).
I did reconstruction with torch and Keras frameworks but I had bigger a error than Caffe (https://github.com/KamitaniLab/DeepImageReconstruction). when I ran this code, I got a big error too. I ran this code and compare its results with the results of Caffe (https://github.com/KamitaniLab/DeepImageReconstruction).
I said my problem to you and you said that this problem may due to my Cuda and Torch versions. I sent my results with your data, my Python and Torch versions, and python code for you. I have big errors. why????
Torch version: 1.12.1+cu116
python version: 3.10.6
result in iter200: error= 2594284032.0 for n01443537_22563.jpeg, sub-01
`
import argparse
import glob
from itertools import product
import os
import pickle
from bdpy.recon.torch.icnn import reconstruct
from bdpy.recon.utils import normalize_image, clip_extreme
from bdpy.dl.torch.models import VGG19, AlexNetGenerator, layer_map
from bdpy.dataform import Features, DecodedFeatures
from bdpy.feature import normalize_feature
from bdpy.util import dump_info
import numpy as np
import PIL.Image
import scipy.io as sio
import torch
print(torch.version)
import torch.optim as optim
import yaml
Functions
def image_preprocess(img, image_mean=np.float32([104, 117, 123])):
'''convert to Caffe's input image layout'''
return np.float32(np.transpose(img, (2, 0, 1))[::-1]) - np.reshape(image_mean, (3, 1, 1))
def image_deprocess(img, image_mean=np.float32([104, 117, 123])):
'''convert from Caffe's input image layout'''
return np.dstack((img + np.reshape(image_mean, (3, 1, 1)))[::-1])
Network settings -------------------------------------------------------
features_dir = '/home/mvl/kalantari/data/decoded_features/ImageNetTest/deeprecon_originals/VGG19'
output_dir = '/home/mvl/kalantari/results/'
subject = 'sub-01'
roi = 'VC'
#device = 'cuda:0'
encoder_param_file = '/home/mvl/kalantari/data/net/VGG_ILSVRC_19_layers/VGG_ILSVRC_19_layers.pt'
layers = [
'conv1_1', 'conv1_2', 'conv2_1', 'conv2_2',
'conv3_1', 'conv3_2', 'conv3_3', 'conv3_4',
'conv4_1', 'conv4_2', 'conv4_3', 'conv4_4',
'conv5_1', 'conv5_2', 'conv5_3', 'conv5_4',
]
layer_mapping = layer_map('vgg19')
encoder_input_shape = (224, 224, 3)
generator_param_file = '/home/mvl/kalantari/data/net/bvlc_reference_caffenet_generator_ILSVRC2012_Training/generator_relu7.pt'
image_mean_file = '/home/mvl/kalantari/data/net/VGG_ILSVRC_19_layers/ilsvrc_2012_mean.npy'
image_mean = np.load(image_mean_file)
image_mean = np.float32([image_mean[0].mean(), image_mean[1].mean(), image_mean[2].mean()])
feature_std_file = '/home/mvl/kalantari/data/net/VGG_ILSVRC_19_layers/estimated_cnn_feat_std_VGG_ILSVRC_19_layers_ImgSize_224x224_chwise_dof1.mat'
feature_range_file = '/home/mvl/kalantari/data/net/bvlc_reference_caffenet_generator_ILSVRC2012_Training/act_range/3x/relu7.txt'
std_ddof = 1
channel_axis = 0
n_iter = 200
# Reconstruction options -------------------------------------------------
opts = {
'loss_func': torch.nn.MSELoss(reduction='sum'),
'n_iter': n_iter,
'lr': (2., 1e-10),
'momentum': (0.9, 0.9),
'decay': (0.01, 0.01),
'blurring': False,
'channels': None,
'masks': None,
'disp_interval': 1,}
Initial image for the optimization (here we use the mean of ilsvrc_2012_mean.npy as RGB values)
initial_image = np.zeros((224, 224, 3), dtype='float32')
initial_image[:, :, 0] = image_mean[2].copy()
initial_image[:, :, 1] = image_mean[1].copy()
initial_image[:, :, 2] = image_mean[0].copy()
Feature SD estimated from true DNN features of 10000 images
feat_std0 = sio.loadmat(feature_std_file)
Feature upper/lower bounds
cols = 4096
up_size = (4096,)
upper_bound = np.loadtxt(feature_range_file,
delimiter=' ',
usecols=np.arange(0, cols),
unpack=True)
upper_bound = upper_bound.reshape(up_size)
Initial features -------------------------------------------------------
initial_gen_feat = np.random.normal(0, 1, (4096,))
Setup results directory ------------------------------------------------
if not os.path.exists(output_dir):
os.makedirs(output_dir)
Set reconstruction options ---------------------------------------------
opts.update({
# The initial image for the optimization (setting to None will use random noise as initial image)
'initial_feature': initial_gen_feat,
'feature_upper_bound': upper_bound,
'feature_lower_bound': 0.,
})
decoded = subject is not None and roi is not None
print('----------------------------------------')
if decoded:
print('Subject: ' + subject)
print('ROI: ' + roi)
print('')
if decoded:
save_dir = os.path.join(output_dir, subject, roi)
else:
save_dir = os.path.join(output_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
Get images if images is None
if decoded:
matfiles = glob.glob(os.path.join(features_dir, layers[0], subject, roi, '.mat'))
else:
matfiles = glob.glob(os.path.join(features_dir, layers[0], '.mat'))
images = [os.path.splitext(os.path.basename(fl))[0] for fl in matfiles]
Load DNN features
if decoded:
features = DecodedFeatures(os.path.join(features_dir), squeeze=False)
else:
features = Features(features_dir)
Images loop
for image_label in images[:1]:
print('Image: ' + image_label)
# Encoder model
encoder = VGG19()
encoder.load_state_dict(torch.load(encoder_param_file))
encoder.eval()
# Generator model
generator = AlexNetGenerator()
generator.load_state_dict(torch.load(generator_param_file))
generator.eval()
# Districuted computation control
snapshots_dir = os.path.join(save_dir, 'snapshots', 'image-%s' % image_label)
if os.path.exists(snapshots_dir):
print('Already done or running. Skipped.')
continue
# Load DNN features
if decoded:
feat = {
layer: features.get(layer=layer, subject=subject, roi=roi, image=image_label)
for layer in layers
}
else:
labels = features.labels
feat = {
layer: features.get_features(layer)[np.array(labels) == image_label]
for layer in layers
}
for layer, ft in feat.items():
ft0 = normalize_feature(
ft[0],
channel_wise_mean=False, channel_wise_std=False,
channel_axis=channel_axis,
shift='self', scale=np.mean(feat_std0[layer]),
std_ddof=std_ddof
)
ft = ft0[np.newaxis]
feat.update({layer: ft})
# Norm of the DNN features for each layer
feat_norm = np.array([np.linalg.norm(feat[layer])
for layer in layers],
dtype='float32')
weights = 1. / (feat_norm ** 2)
# Normalise the weights such that the sum of the weights = 1
weights = weights / weights.sum()
layer_weights = dict(zip(layers, weights))
opts.update({'layer_weights': layer_weights})
# Reconstruction
snapshots_dir = os.path.join(save_dir, 'snapshots', 'image-%s' % image_label)
recon_image, loss_list = reconstruct(feat,
encoder,
generator=generator,
layer_mapping=layer_mapping,
optimizer=optim.SGD,
image_size=encoder_input_shape,
crop_generator_output=True,
preproc=image_preprocess,
postproc=image_deprocess,
output_dir=save_dir,
save_snapshot=True,
snapshot_dir=snapshots_dir,
snapshot_ext='tiff',
snapshot_postprocess=normalize_image,
return_loss=True,
**opts)
# Save the raw reconstructed image
recon_image_mat_file = os.path.join(save_dir, 'recon_image' + '-' + image_label + '.mat')
sio.savemat(recon_image_mat_file, {'recon_image': recon_image})
recon_image_normalized_file = os.path.join(save_dir, 'recon_image_normalized' + '-' + image_label + '.tiff')
PIL.Image.fromarray(normalize_image(clip_extreme(recon_image, pct=4))).save(recon_image_normalized_file)
print('All done')`