omg_empathy/process_data_trainset.py at master · rosielab/omg_empathy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import os
import csv
import numpy as np
#from keras.applications.mobilenet import DepthwiseConv2D
from keras.layers import DepthwiseConv2D
from keras.models import load_model
from keras.preprocessing import image
IMAGE_SIZE = 128

def get_regressor_predictions(model, paths):
    valence = []
    for i, path in enumerate(paths):
        img = image.load_img(path, target_size=(IMAGE_SIZE, IMAGE_SIZE))
        img = image.img_to_array(img) / 255
        img = np.array(img).reshape((1, IMAGE_SIZE, IMAGE_SIZE, 3))
        p = model.predict(img)
        valence.append(p[0][0])
    return valence_p

def get_classifier_predictions(model, paths):
    H_r= [] #matching matrix I am looking for
    for i, path in enumerate(paths):
        img = image.load_img(path, target_size=(IMAGE_SIZE, IMAGE_SIZE))
        img = image.img_to_array(img) / 255
        img = np.array(img).reshape((1, IMAGE_SIZE, IMAGE_SIZE, 3))
        p = model.predict(img)
        H_r.append(p[0, 1]) #p[0,1] for happy and p[0,0] fo neutral
    return H_r


def get_name (folder):
    #get file name
    dataList = os.listdir(folder)
    name_list = []
    for i in range(len(dataList)):
        name_list.append(dataList[i].split('.')[0])
    return name_list


file = open('./Frames_Count.txt','r').read().splitlines()
file_name = []
framelist = []
for line in file:
    if line != '':
        name = line.split('-')[0]
        frame_n = line.split('-')[1]
        file_name.append(name.split('.')[0])
        framelist.append(int(frame_n.split(' ')[1]))


#path = '../../OMGEmpathyChallenge-master/data/labels/Training/'
img_path = '../../OMGEmpathyChallenge-master/data/faces/test/'
#img_path = '../../OMGEmpathyChallenge-master/data/faces/Training/'
#save_path = '../../OMGEmpathyChallenge-master/data/prediction/Validation/'
#save_path = '../../OMGEmpathyChallenge-master/data/temp/'
save_path = '../../OMGEmpathyChallenge-master/data/test/'
frame_path = '../../OMGEmpathyChallenge-master/data/testmatch/'
#file_name = get_name(path)
r_path = './CNN/regressor.h5'
c_path = './CNN/classifier.h5'
count = 0
for i in range(len(file_name)):
    filename = file_name[i]
    if filename == '':
        pass
    else:
        count = count + 1
        print('== COUNT {}=='.format(count))
        print('== PROCESSING {} =='.format(filename))
        #label_file = path + '{}.csv'.format(filename)
        img_folder = img_path + '{}.mp4'.format(filename)
        predfile = save_path + '{}.csv'.format(filename)
        framefile = frame_path + '{}.csv'.format(filename)
        truth = []
        frame = framelist[i]
        v_paths = []
        for i in range(0,frame):
            if i % 25 == 0:
                ipath = img_folder + '/Subject/{}.png'.format(i)
                v_paths.append(ipath)
        #print('== CALCULATING ==')
        model = load_model(r_path, custom_objects={'DepthwiseConv2D': DepthwiseConv2D})
        valence_p = get_regressor_predictions(model, v_paths)
        #print('== SAVING RESULTS ==')
        with open(predfile, "w") as output:
            row = 0
            writer = csv.writer(output, lineterminator='\n')
            for val in valence_p:
                writer.writerow([val])
                row = row + 1
        #print('== Done ==')
        #print('== Matching happy face ==')
        v_paths_a = []
        for i in range(0,frame):
            if i % 25 == 0:
                ipath = img_folder + '/Actor/{}.png'.format(i)
                v_paths_a.append(ipath)
        #print('== CALCULATING FIRST ONE ==')
        model = load_model(c_path, custom_objects={'DepthwiseConv2D': DepthwiseConv2D})
        H_r_a = get_classifier_predictions(model, v_paths_a)
        #print('== CALCULATING SECOND ONE==')
        H_r_s = get_classifier_predictions(model, v_paths)
        match_r = []
        for i in range(0,len(H_r_s)):
            ar_t = H_r_a[i]
            sr_t = H_r_s[i]
            if ar_t > 0.25 and sr_t > 0.25:
                match_r.append(1)
            else:
                match_r.append(0)
        matched_frame = []
        for i in range(0,len(match_r)):
            if match_r[i] == 1:
                matched_frame.append(i)
        #print('== SAVING MATCHED FRAMES TO CSV FILE ==')
        with open(framefile, "w") as output:
            writer = csv.writer(output, lineterminator='\n')
            for val in matched_frame:
                writer.writerow([val])
        #print('== NEXT FILE ==')
print('== FINISHED ==')