Add files using upload-large-folder tool

.gitattributes

@@ -2939,3 +2939,48 @@ results/versatile_diffusion/subj01/956.png filter=lfs diff=lfs merge=lfs -text
 results/versatile_diffusion/subj01/948.png filter=lfs diff=lfs merge=lfs -text
 results/versatile_diffusion/subj01/954.png filter=lfs diff=lfs merge=lfs -text
 results/versatile_diffusion/subj01/946.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/952.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/955.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/951.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/962.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/959.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/973.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/957.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/970.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/977.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/976.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/972.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/969.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/974.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/975.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/978.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/961.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/953.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/966.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/965.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/968.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/964.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/963.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/960.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/971.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/981.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/979.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/967.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/958.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/96.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/99.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/980.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/0.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/10.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/1.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/5.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/7.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/6.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/11.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/98.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/9.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/8.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/97.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/12.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/2.png filter=lfs diff=lfs merge=lfs -text
+results/versatile_diffusion/subj01/roi/3.png filter=lfs diff=lfs merge=lfs -text

scripts/clipvision_extract_features.py

@@ -0,0 +1,88 @@
+import sys
+sys.path.append('versatile_diffusion')
+import os
+import PIL
+from PIL import Image
+import numpy as np
+
+import torch
+from lib.cfg_helper import model_cfg_bank
+from lib.model_zoo import get_model
+from lib.experiments.sd_default import color_adjust, auto_merge_imlist
+from torch.utils.data import DataLoader, Dataset
+
+from lib.model_zoo.vd import VD
+from lib.cfg_holder import cfg_unique_holder as cfguh
+from lib.cfg_helper import get_command_line_args, cfg_initiates, load_cfg_yaml
+import torchvision.transforms as T
+
+import argparse
+parser = argparse.ArgumentParser(description='Argument Parser')
+parser.add_argument("-sub", "--sub",help="Subject Number",default=1)
+args = parser.parse_args()
+sub=int(args.sub)
+assert sub in [1,2,5,7]
+
+cfgm_name = 'vd_noema'
+
+pth = 'versatile_diffusion/pretrained/vd-four-flow-v1-0-fp16-deprecated.pth'
+cfgm = model_cfg_bank()(cfgm_name)
+net = get_model()(cfgm)
+sd = torch.load(pth, map_location='cpu')
+net.load_state_dict(sd, strict=False)    
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+net.clip = net.clip.to(device)
+
+class batch_generator_external_images(Dataset):
+
+    def __init__(self, data_path):
+        self.data_path = data_path
+        self.im = np.load(data_path).astype(np.uint8)
+
+
+    def __getitem__(self,idx):
+        img = Image.fromarray(self.im[idx])
+        img = T.functional.resize(img,(512,512))
+        img = T.functional.to_tensor(img).float()
+        #img = img/255
+        img = img*2 - 1
+        return img
+
+    def __len__(self):
+        return  len(self.im)
+    
+batch_size=1
+image_path = 'data/processed_data/subj{:02d}/nsd_train_stim_sub{}.npy'.format(sub,sub)
+train_images = batch_generator_external_images(data_path = image_path)
+
+image_path = 'data/processed_data/subj{:02d}/nsd_test_stim_sub{}.npy'.format(sub,sub)
+test_images = batch_generator_external_images(data_path = image_path)
+
+trainloader = DataLoader(train_images,batch_size,shuffle=False)
+testloader = DataLoader(test_images,batch_size,shuffle=False)
+
+num_embed, num_features, num_test, num_train = 257, 768, len(test_images), len(train_images)
+
+train_clip = np.zeros((num_train,num_embed,num_features))
+test_clip = np.zeros((num_test,num_embed,num_features))
+
+with torch.no_grad():
+    for i,cin in enumerate(testloader):
+        print(i)
+        #ctemp = cin*2 - 1
+        c = net.clip_encode_vision(cin)
+        test_clip[i] = c[0].cpu().numpy()
+    
+    np.save('data/extracted_features/subj{:02d}/nsd_clipvision_test.npy'.format(sub),test_clip)
+        
+    for i,cin in enumerate(trainloader):
+        print(i)
+        #ctemp = cin*2 - 1
+        c = net.clip_encode_vision(cin)
+        train_clip[i] = c[0].cpu().numpy()
+    np.save('data/extracted_features/subj{:02d}/nsd_clipvision_train.npy'.format(sub),train_clip)
+
+
+
+    

scripts/clipvision_regression.py

@@ -0,0 +1,71 @@
+import sys
+import numpy as np
+import sklearn.linear_model as skl
+import pickle
+import argparse
+parser = argparse.ArgumentParser(description='Argument Parser')
+parser.add_argument("-sub", "--sub",help="Subject Number",default=1)
+args = parser.parse_args()
+sub=int(args.sub)
+assert sub in [1,2,5,7]
+
+train_path = 'data/processed_data/subj{:02d}/nsd_train_fmriavg_nsdgeneral_sub{}.npy'.format(sub,sub)
+train_fmri = np.load(train_path)
+test_path = 'data/processed_data/subj{:02d}/nsd_test_fmriavg_nsdgeneral_sub{}.npy'.format(sub,sub)
+test_fmri = np.load(test_path)
+
+## Preprocessing fMRI
+
+train_fmri = train_fmri/300
+test_fmri = test_fmri/300
+
+
+norm_mean_train = np.mean(train_fmri, axis=0)
+norm_scale_train = np.std(train_fmri, axis=0, ddof=1)
+train_fmri = (train_fmri - norm_mean_train) / norm_scale_train
+test_fmri = (test_fmri - norm_mean_train) / norm_scale_train
+
+print(np.mean(train_fmri),np.std(train_fmri))
+print(np.mean(test_fmri),np.std(test_fmri))
+
+print(np.max(train_fmri),np.min(train_fmri))
+print(np.max(test_fmri),np.min(test_fmri))
+
+num_voxels, num_train, num_test = train_fmri.shape[1], len(train_fmri), len(test_fmri)
+
+
+train_clip = np.load('data/extracted_features/subj{:02d}/nsd_clipvision_train.npy'.format(sub))
+test_clip = np.load('data/extracted_features/subj{:02d}/nsd_clipvision_test.npy'.format(sub))
+
+#train_clip = train_clip[:,1:,:]
+num_samples,num_embed,num_dim = train_clip.shape
+
+print("Training Regression")
+reg_w = np.zeros((num_embed,num_dim,num_voxels)).astype(np.float32)
+reg_b = np.zeros((num_embed,num_dim)).astype(np.float32)
+pred_clip = np.zeros_like(test_clip)
+for i in range(num_embed):
+
+
+    reg = skl.Ridge(alpha=60000, max_iter=50000, fit_intercept=True)
+    reg.fit(train_fmri, train_clip[:,i])
+    reg_w[i] = reg.coef_
+    reg_b[i] = reg.intercept_
+    
+    pred_test_latent = reg.predict(test_fmri)
+    std_norm_test_latent = (pred_test_latent - np.mean(pred_test_latent,axis=0)) / np.std(pred_test_latent,axis=0)
+    pred_clip[:,i] = std_norm_test_latent * np.std(train_clip[:,i],axis=0) + np.mean(train_clip[:,i],axis=0)
+    
+    print(i,reg.score(test_fmri,test_clip[:,i]))
+    
+
+np.save('data/predicted_features/subj{:02d}/nsd_clipvision_predtest_nsdgeneral.npy'.format(sub),pred_clip)
+
+datadict = {
+    'weight' : reg_w,
+    'bias' : reg_b,
+
+}
+
+with open('data/regression_weights/subj{:02d}/clipvision_regression_weights.pkl'.format(sub),"wb") as f:
+  pickle.dump(datadict,f)

scripts/eval_extract_features.py

@@ -0,0 +1,147 @@
+import os
+import sys
+import numpy as np
+import h5py
+import scipy.io as spio
+import nibabel as nib
+
+import torch
+import torchvision
+import torchvision.models as tvmodels
+import torchvision.transforms as transforms
+from torch.utils.data import DataLoader, Dataset
+import torchvision.transforms as T
+from PIL import Image
+import clip
+
+import skimage.io as sio
+from skimage import data, img_as_float
+from skimage.transform import resize as imresize
+from skimage.metrics import structural_similarity as ssim
+import scipy as sp
+
+import argparse
+parser = argparse.ArgumentParser(description='Argument Parser')
+parser.add_argument("-sub", "--sub", help="Subject Number", default=1)
+args = parser.parse_args()
+sub = int(args.sub)
+assert sub in [0, 1, 2, 5, 7]
+
+images_dir = 'data/nsddata_stimuli/test_images'
+feats_dir = 'data/eval_features/test_images'
+
+if sub in [1, 2, 5, 7]:
+    feats_dir = f'data/eval_features/subj{sub:02d}'
+    images_dir = f'results/versatile_diffusion/subj{sub:02d}'
+
+if not os.path.exists(feats_dir):
+    os.makedirs(feats_dir)
+
+class batch_generator_external_images(Dataset):
+    def __init__(self, data_path='', prefix='', net_name='clip'):
+        self.data_path = data_path
+        self.prefix = prefix
+        self.net_name = net_name
+
+        if self.net_name == 'clip':
+            self.normalize = transforms.Normalize(
+                mean=[0.48145466, 0.4578275, 0.40821073],
+                std=[0.26862954, 0.26130258, 0.27577711]
+            )
+        else:
+            self.normalize = transforms.Normalize(
+                mean=[0.485, 0.456, 0.406],
+                std=[0.229, 0.224, 0.225]
+            )
+        self.num_test = 982
+
+    def __getitem__(self, idx):
+        img = Image.open(f'{self.data_path}/{self.prefix}{idx}.png')
+        img = T.functional.resize(img, (224, 224))
+        img = T.functional.to_tensor(img).float()
+        img = self.normalize(img)
+        return img
+
+    def __len__(self):
+        return self.num_test
+
+# Set device
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+global feat_list
+feat_list = []
+
+def fn(module, inputs, outputs):
+    feat_list.append(outputs.cpu().numpy())
+
+net_list = [
+    ('inceptionv3', 'avgpool'),
+    ('clip', 'final'),
+    ('alexnet', 2),
+    ('alexnet', 5),
+    ('efficientnet', 'avgpool'),
+    ('swav', 'avgpool')
+]
+
+batchsize = 64
+
+for (net_name, layer) in net_list:
+    feat_list = []
+    print(net_name, layer)
+
+    dataset = batch_generator_external_images(data_path=images_dir, net_name=net_name, prefix='')
+    loader = DataLoader(dataset, batchsize, shuffle=False)
+
+    if net_name == 'inceptionv3':
+        net = tvmodels.inception_v3(pretrained=True)
+        if layer == 'avgpool':
+            net.avgpool.register_forward_hook(fn)
+        elif layer == 'lastconv':
+            net.Mixed_7c.register_forward_hook(fn)
+
+    elif net_name == 'alexnet':
+        net = tvmodels.alexnet(pretrained=True)
+        if layer == 2:
+            net.features[4].register_forward_hook(fn)
+        elif layer == 5:
+            net.features[11].register_forward_hook(fn)
+        elif layer == 7:
+            net.classifier[5].register_forward_hook(fn)
+
+    elif net_name == 'clip':
+        model, _ = clip.load("ViT-L/14", device=device)
+        net = model.visual.to(torch.float32)
+        if layer == 7:
+            net.transformer.resblocks[7].register_forward_hook(fn)
+        elif layer == 12:
+            net.transformer.resblocks[12].register_forward_hook(fn)
+        elif layer == 'final':
+            net.register_forward_hook(fn)
+
+    elif net_name == 'efficientnet':
+        net = tvmodels.efficientnet_b1(weights='IMAGENET1K_V1')
+        net.avgpool.register_forward_hook(fn)
+
+    elif net_name == 'swav':
+        net = torch.hub.load('facebookresearch/swav:main', 'resnet50')
+        net.avgpool.register_forward_hook(fn)
+
+    net.eval()
+    net = net.to(device)
+
+    with torch.no_grad():
+        for i, x in enumerate(loader):
+            print(i * batchsize)
+            x = x.to(device)
+            _ = net(x)
+
+    if net_name == 'clip':
+        if layer == 7 or layer == 12:
+            feat_list = np.concatenate(feat_list, axis=1).transpose((1, 0, 2))
+        else:
+            feat_list = np.concatenate(feat_list)
+    else:
+        feat_list = np.concatenate(feat_list)
+
+    file_name = f'{feats_dir}/{net_name}_{layer}.npy'
+    np.save(file_name, feat_list)

scripts/evaluate_reconstruction.py

@@ -0,0 +1,93 @@
+import os
+import sys
+import numpy as np
+import h5py
+import scipy.io as spio
+import nibabel as nib
+import scipy as sp
+from PIL import Image
+
+
+
+import argparse
+parser = argparse.ArgumentParser(description='Argument Parser')
+parser.add_argument("-sub", "--sub",help="Subject Number",default=1)
+args = parser.parse_args()
+sub=int(args.sub)
+assert sub in [1,2,5,7]
+
+
+from scipy.stats import pearsonr,binom,linregress
+import numpy as np
+def pairwise_corr_all(ground_truth, predictions):
+    r = np.corrcoef(ground_truth, predictions)#cosine_similarity(ground_truth, predictions)#
+    r = r[:len(ground_truth), len(ground_truth):]  # rows: groundtruth, columns: predicitons
+    #print(r.shape)
+    # congruent pairs are on diagonal
+    congruents = np.diag(r)
+    #print(congruents)
+    
+    # for each column (predicition) we should count the number of rows (groundtruth) that the value is lower than the congruent (e.g. success).
+    success = r < congruents
+    success_cnt = np.sum(success, 0)
+    
+    # note: diagonal of 'success' is always zero so we can discard it. That's why we divide by len-1
+    perf = np.mean(success_cnt) / (len(ground_truth)-1)
+    p = 1 - binom.cdf(perf*len(ground_truth)*(len(ground_truth)-1), len(ground_truth)*(len(ground_truth)-1), 0.5)
+    
+    return perf, p
+
+
+net_list = [
+    ('inceptionv3','avgpool'),
+    ('clip','final'),
+    ('alexnet',2),
+    ('alexnet',5),
+    ('efficientnet','avgpool'),
+    ('swav','avgpool')
+    ]
+
+feats_dir = 'data/eval_features/subj{:02d}'.format(sub)
+test_dir = 'data/eval_features/test_images'
+num_test = 982
+distance_fn = sp.spatial.distance.correlation
+pairwise_corrs = []
+for (net_name,layer) in net_list:
+    file_name = '{}/{}_{}.npy'.format(test_dir,net_name,layer)
+    gt_feat = np.load(file_name)
+    
+    file_name = '{}/{}_{}.npy'.format(feats_dir,net_name,layer)
+    eval_feat = np.load(file_name)
+    
+    gt_feat = gt_feat.reshape((len(gt_feat),-1))
+    eval_feat = eval_feat.reshape((len(eval_feat),-1))
+    
+    print(net_name,layer)
+    if net_name in ['efficientnet','swav']:
+        print('distance: ',np.array([distance_fn(gt_feat[i],eval_feat[i]) for i in range(num_test)]).mean())
+    else:
+        pairwise_corrs.append(pairwise_corr_all(gt_feat[:num_test],eval_feat[:num_test])[0])
+        print('pairwise corr: ',pairwise_corrs[-1])
+        
+from skimage.color import rgb2gray
+from skimage.metrics import structural_similarity as ssim
+        
+ssim_list = []
+pixcorr_list = []
+for i in range(982):
+    gen_image = Image.open('results/versatile_diffusion/subj{:02d}/{}.png'.format(sub,i)).resize((425,425))
+    gt_image = Image.open('data/nsddata_stimuli/test_images/{}.png'.format(i))
+    gen_image = np.array(gen_image)/255.0
+    gt_image = np.array(gt_image)/255.0
+    pixcorr_res = np.corrcoef(gt_image.reshape(1,-1), gen_image.reshape(1,-1))[0,1]
+    pixcorr_list.append(pixcorr_res)
+    gen_image = rgb2gray(gen_image)
+    gt_image = rgb2gray(gt_image)
+    ssim_res = ssim(gen_image, gt_image, multichannel=True, gaussian_weights=True, sigma=1.5, use_sample_covariance=False, data_range=1.0)
+    ssim_list.append(ssim_res)
+    
+ssim_list = np.array(ssim_list)
+pixcorr_list = np.array(pixcorr_list)
+print('PixCorr: {}'.format(pixcorr_list.mean()))
+print('SSIM: {}'.format(ssim_list.mean()))
+