tesi/ChatManager.py at master · ManofWax/tesi · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
__author__ = 'wax'
import re
import nltk
import logging
import time
import os

class ChatCorpus:
    """
    Handle processed chat files like a corpus
    """
    def __init__(self, file_directory_path, file_name):
        #self.wordlists = nltk.PlaintextCorpusReader(file_directory_path, file_extension)
        self.filename = file_name
        file = open(file_name, encoding="utf8")
        self.file_txt = file.read()
        file.close()

    def ExtractEmoticons(self):
        kappa = re.compile(".*Kappa.*\n")
        file_kappa = " ".join(re.findall(kappa, self.file_txt))
        file = open(self.filename + "Kappa",mode='w', encoding="utf8")
        file.write(file_kappa)
        file.close()

class ChatManager:
    """
    Process raw files and save them
    """
    def __init__(self, file_path):
        self.filePath = file_path
        self.processedFile = ""
        file = open(file_path, encoding="utf8")
        self.rawFile = file.read()
        self.nonAsciiLines = ""
        file.close()

    def process_file(self):
        """
        Remove shit from the raw file
        :return:
        """
        rep = re.compile(r"""
                        http[s]?://.*?[\s|\n]
                        |www.*?[\s|\n]
                        |(\n){2,}
                        """, re.X)
        non_asc = re.compile(".*[^\x00-\x7F].*\n")
        start_time = time.time()
        logging.debug("Start file processing")
        self.processedFile = re.sub(rep, "", self.rawFile)
        self.processedFile = re.sub(non_asc, "", self.processedFile)
        end_time = time.time()
        logging.debug("Processed file in " + str(end_time - start_time))
        start_time = time.time()
        logging.debug("Start finding non-ASCII lines")
        self.nonAsciiLines = " ".join(re.findall(non_asc, self.rawFile))
        end_time = time.time()
        logging.debug("Lines found in " + str(end_time - start_time))

    def print_statistics(self):
        print("Original file lenght:{} \n Processed file lenght:{}\n NonAscii lenght: {}".format(len(self.rawFile), len(self.processedFile), len(self.nonAsciiLines)))

    def save(self, file_path):
        """
        Save the processed file
        :param file_path: the save path
        :return:
        """
        file = open(file_path, mode='w', encoding="utf8")
        file.write(self.processedFile)
        logging.debug("Saved file " + file_path)
        file.close()
        file = open(file_path + "notAscii", mode='w', encoding="utf8")
        file.write(self.nonAsciiLines)
        logging.debug("Saved file 2")
        file.close()

class ChatLive:
    """Classe di analisi del testo per le chat live"""
    def __index__(self):
        pass