BigData2-Proj3-Frame/shop_keywords_extract.py at master · Inf4Moment/BigData2-Proj3-Frame · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
# from pyhanlp import *
import os
import json

# CoreStopWordDictionary = SafeJClass("com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary")
# NLPTokenizer = JClass("com.hankcs.hanlp.tokenizer.NLPTokenizer")
# Nature = SafeJClass("com.hankcs.hanlp.corpus.tag.Nature")
# ClusterAnalyzer = SafeJClass('com.hankcs.hanlp.mining.cluster.ClusterAnalyzer')
# reserve_words_feature = [Nature.ns, Nature.n, Nature.vn, Nature.nr]

# analyzer_item = ClusterAnalyzer()
# analyzer_shop = ClusterAnalyzer()
shop_set = {}
item_set = {}
key_set = {}

files = os.listdir(r"../text")
for file_name in files:
    file_path = r"../text/" + file_name
    key_text = ''
    shop_text = ''
    item_text = ''
    item_flag = False
    shop_flag = False
    for line in open(file_path, "r", encoding="utf-8"):
        if '-------' in line:
            item_text = ''
            shop_text = ''
            item_flag = True
            shop_flag = True
        elif '+++++++' in line:
            item_text += shop_text
            shop_flag = False
        elif '=======' in line:
            key_text += item_text
            item_set[item_text.split('\n')[0]] = item_text
            # analyzer_item.addDocument(item_text.split('\n')[0], item_text)
            item_flag = False
        else:
            if shop_flag:
                shop_text += line + ','
            elif item_flag:
                item_text += line + ','
            else:
                key_text += line + ','

    shop_text += key_text.split(',')[0] + ','

    print('Extracting\t' + file_name + '\tKeyword...')

    # filter_text = ''
    # text_seg = HanLP.segment(key_text)
    # CoreStopWordDictionary.apply(text_seg)
    # for word in text_seg:
    #     if word.nature in reserve_words_feature:
    #         filter_text += word.word + '\n'
    # keyword_list = HanLP.extractKeyword(filter_text, 5)
    #
    # py_key_list = []
    # for key in keyword_list:
    #     py_key_list.append(key)
    # key_set[file_name.split('.')[0]] = py_key_list
    #
    # shop_text += ','.join(py_key_list*5)
    # shop_set[file_name.split('.')[0]] = shop_text
    # analyzer_shop.addDocument(file_name.split('.')[0], shop_text)

output_path = './res/keyword.json'
out = open(output_path, 'w', encoding='utf-8')
# out.write('\n'.join(keyword_list))
json.dump(key_set, out, indent=4, ensure_ascii=False)
out.close()

print('Clustering\tShops...')

# clusters = analyzer_shop.repeatedBisection(16)
# cluster_list = []
# for cluster in clusters:
#     py_cluster = []
#     for item in cluster:
#         py_cluster.append(item)
#     cluster_text = ''
#     for item in py_cluster:
#         cluster_text += shop_set[item]
#
#     filter_text = ''
#     text_seg = HanLP.segment(cluster_text)
#     CoreStopWordDictionary.apply(text_seg)
#     for word in text_seg:
#         if word.nature in reserve_words_feature:
#             filter_text += word.word + '\n'
#
#     keyword_list = HanLP.extractKeyword(filter_text, 3)
#     py_key_list = []
#     for key in keyword_list:
#         py_key_list.append(key)
#
#     cluster_set = {
#         'key_word': py_key_list,
#         'shop_list': py_cluster,
#     }
#     cluster_list.append(cluster_set)
# output_path = './res/shop_cluster.json'
# out = open(output_path, 'w', encoding='utf-8')
# json.dump(cluster_list, out, indent=4, ensure_ascii=False)
# out.close()

print('Clustering\tShops\tOK...')

print('Clustering\tItems...')

# clusters = analyzer_item.repeatedBisection(16)
# cluster_list = []
# for cluster in clusters:
#     py_cluster = []
#     for item in cluster:
#         py_cluster.append(item)
#     cluster_text = ''
#     for item in py_cluster:
#         cluster_text += item_set[item]
#
#     filter_text = ''
#     text_seg = HanLP.segment(cluster_text)
#     CoreStopWordDictionary.apply(text_seg)
#     for word in text_seg:
#         if word.nature in reserve_words_feature:
#             filter_text += word.word + '\n'
#
#     keyword_list = HanLP.extractKeyword(filter_text, 3)
#     py_key_list = []
#     for key in keyword_list:
#         py_key_list.append(key)
#
#     cluster_set = {
#         'key_word': py_key_list,
#         'item_list': py_cluster,
#     }
#     cluster_list.append(cluster_set)
# output_path = './res/item_cluster.json'
# out = open(output_path, 'w', encoding='utf-8')
# json.dump(cluster_list, out, indent=4, ensure_ascii=False)
# out.close()

print('Clustering\tItems\tOK...')