Caption-It/Caption_it.py at main · Akamourya18/Caption-It · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/usr/bin/env python
# coding: utf-8

# In[1]:


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import keras
import json
import pickle
from keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input, decode_predictions
from tensorflow.keras.preprocessing import image
from keras.models import Model, load_model
from tensorflow.keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.layers import Input, Dense, Dropout, Embedding, LSTM
from tensorflow.keras.layers import add


# In[2]:


model = load_model("D:\Caption It\model_weights\model_9.h5")


# In[3]:


model_temp = ResNet50(weights="imagenet",input_shape=(224,224,3))


# In[4]:


model_resnet = Model(model_temp.input,model_temp.layers[-2].output)


# In[8]:


def preprocess_img(img):
    img = image.load_img(img,target_size=(224,224))
    img = image.img_to_array(img)
    img = np.expand_dims(img,axis=0)
    # Normalisation
    img = preprocess_input(img)
    return img


# In[23]:


def encode_image(img):
    img = preprocess_img(img)
    feature_vector = model_resnet.predict(img)

    feature_vector = feature_vector.reshape((1, feature_vector.shape[1]))
    #print(feature_vector.shape)
    return feature_vector


# In[24]:


with open("D:\Caption It\storage\word_to_idx.pkl", "rb") as w2i:
    word_to_idx = pickle.load(w2i)
with open("D:\Caption It\storage\idx_to_word.pkl", "rb") as i2w:
    idx_to_word = pickle.load(i2w)


# In[25]:


word_to_idx


# In[26]:


def predict_caption(photo):
    max_len=35
    in_text = "startseq"
    for i in range(max_len):
        sequence = [word_to_idx[w] for w in in_text.split() if w in word_to_idx]
        sequence = pad_sequences([sequence],maxlen=max_len,padding='post')

        ypred = model.predict([photo,sequence])
        ypred = ypred.argmax() #WOrd with max prob always - Greedy Sampling
        word = idx_to_word[ypred]
        in_text += (' ' + word)

        if word == "endseq":
            break

    final_caption = in_text.split()[1:-1]
    final_caption = ' '.join(final_caption)
    return final_caption


# In[27]:

def caption_this_image(image):
    enc = encode_image(image)
    caption = predict_caption(enc)
    return caption


# In[ ]: