sentiment-analysis/emotional.py at master · jimenbian/sentiment-analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import

import tflearn
from tflearn.data_utils import to_categorical, pad_sequences
import string
import numpy as nm
import codecs
import re
import collections
import math
import tensorflow as tf
import random
import glob

allWords = []
allDocuments = []
allLabels = []

def readFile(fileName, allWords):


    file = codecs.open(fileName, encoding='utf-8')

    for line in file:
        line = line.lower().encode('utf-8')
        words = line.split()
        for word in words:
            word = word.translate(None, string.punctuation)
            if word != '':
                allWords.append(word)

    file.close()


def readFileToConvertWordsToIntegers(dictionary, fileName, allDocuments, allLabels, label):

    file = codecs.open(fileName, encoding='utf-8')
    document = []
    for line in file:
        line = line.lower().encode('utf-8')
        words = line.split()
        for word in words:
            word = word.translate(None, string.punctuation)
            if word in dictionary:
                index = dictionary[word]
            else:
                index = 0  # dictionary['UNK']
            document.append(index)
        allDocuments.append(document)
        allLabels.append(label)

    file.close()


vocabulary_size = 10000

def build_dataset(words):
  count = [['UNK', -1]]
  count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
  dictionary = dict()
  for word, _ in count:
    dictionary[word] = len(dictionary)
  data = list()
  unk_count = 0
  for word in words:
    if word in dictionary:
      index = dictionary[word]
    else:
      index = 0  # dictionary['UNK']
      unk_count = unk_count + 1
    data.append(index)
  count[0][1] = unk_count
  reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
  return dictionary, reverse_dictionary


fileList = glob.glob("train/neg/*.txt")
for file in fileList:
    readFile(file, allWords)

fileList = glob.glob("train/pos/*.txt")
for file in fileList:
    readFile(file, allWords)

print(len(allWords))

dictionary, reverse_dictionary = build_dataset(allWords)
del allWords  # Hint to reduce memory.

print(len(dictionary))

fileList = glob.glob("train/neg/*.txt")
for file in fileList:
    readFileToConvertWordsToIntegers(dictionary, file, allDocuments, allLabels, 0)

fileList = glob.glob("train/pos/*.txt")
for file in fileList:
    readFileToConvertWordsToIntegers(dictionary, file, allDocuments, allLabels, 1)

print(len(allDocuments))
print(len(allLabels))

c = list(zip(allDocuments, allLabels)) # shuffle them partitioning

random.shuffle(c)

allDocuments, allLabels = zip(*c)
print('a')

trainX = allDocuments[:25]
testX = allDocuments[25:]

trainY = allLabels[:25]
testY = allLabels[25:]


#counter=collections.Counter(trainY)
#print(counter)


trainX = pad_sequences(trainX, maxlen=100, value=0.)
testX = pad_sequences(testX, maxlen=100, value=0.)
# Converting labels to binary vectors
trainY = to_categorical(trainY, nb_classes=2)
testY = to_categorical(testY, nb_classes=2)

# Network building
net = tflearn.input_data([None, 100])
net = tflearn.embedding(net, input_dim=vocabulary_size, output_dim=128)
net = tflearn.lstm(net, 128, dropout=0.8)
net = tflearn.fully_connected(net, 2, activation='softmax')
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
                         loss='categorical_crossentropy')

# Training
model = tflearn.DNN(net, tensorboard_verbose=0)
model.fit(trainX, trainY,n_epoch=1, validation_set=(testX, testY), show_metric=True,
          batch_size=32)
predictions = model.predict(trainX)
print(predictions)