3
3
import time
4
4
import hashlib
5
5
6
- storage1 = {}
6
+ storage1 = {}
7
7
storage2 = {}
8
8
9
+
9
10
def cleanHash (toHash ):
10
11
return hashlib .sha224 (toHash ).hexdigest ()
11
12
12
- def sec_start (job ,header ):
13
+
14
+ def sec_start (job , header ):
13
15
print ('section_start\r \033 [0K' + header )
14
16
17
+
15
18
def sec_end (job ):
16
19
print ('section_end\r \033 [0K' )
17
20
18
- with open (sys .argv [1 ],'r' ) as f :
21
+
22
+ with open (sys .argv [1 ], 'r' ) as f :
19
23
data = json .load (f )
20
24
for message in data ['messages' ]:
21
25
mtyp = message ['type' ].encode ('utf-8' , 'ignore' )
22
26
murl = message ['url' ].encode ('utf-8' , 'ignore' )
23
27
mmes = message ['message' ].encode ('utf-8' , 'ignore' )
24
28
if mtyp not in storage1 .keys ():
25
- storage1 [mtyp ] = {"messages" :{}, "cnt" :0 }
26
- storage2 [mtyp ] = {"urls" :{}, "cnt" :0 }
29
+ storage1 [mtyp ] = {"messages" : {}, "cnt" : 0 }
30
+ storage2 [mtyp ] = {"urls" : {}, "cnt" : 0 }
27
31
if mmes not in storage1 [mtyp ]["messages" ].keys ():
28
- storage1 [mtyp ]["messages" ][mmes ] = {"urls" :{}, "cnt" :0 }
32
+ storage1 [mtyp ]["messages" ][mmes ] = {"urls" : {}, "cnt" : 0 }
29
33
if murl not in storage2 [mtyp ]["urls" ].keys ():
30
- storage2 [mtyp ]["urls" ][murl ] = {"messages" :{}, "cnt" :0 }
34
+ storage2 [mtyp ]["urls" ][murl ] = {"messages" : {}, "cnt" : 0 }
31
35
if murl not in storage1 [mtyp ]["messages" ][mmes ]["urls" ].keys ():
32
36
storage1 [mtyp ]["messages" ][mmes ]["urls" ][murl ] = 0
33
37
if mmes not in storage2 [mtyp ]["urls" ][murl ]["messages" ].keys ():
@@ -38,18 +42,18 @@ def sec_end(job):
38
42
storage2 [mtyp ]["urls" ][murl ]["messages" ][mmes ] += 1
39
43
storage2 [mtyp ]["urls" ][murl ]["cnt" ] += 1
40
44
storage2 [mtyp ]["cnt" ] += 1
41
- # Stats
42
- for key ,value in sorted (storage1 .items (), key = lambda x :x [1 ]['cnt' ]):
43
- print ("Type: {}, Totalfound: {}" .format (key , value ["cnt" ]))
44
- for key2 ,value2 in sorted (storage1 [key ]["messages" ].items (), key = lambda x :x [1 ]['cnt' ], reverse = True ):
45
+
46
+ for key , value in sorted (storage1 .items (), key = lambda x : x [1 ]['cnt' ]):
47
+ print ("Type: {}, Totalfound: {}" .format (key , value ["cnt" ]))
48
+ for key2 , value2 in sorted (storage1 [key ]["messages" ].items (), key = lambda x : x [1 ]['cnt' ], reverse = True ):
45
49
sec_start (key + key2 , key2 )
46
- print ("[{}] [{}%] Message: {}" .format (key , round (100 * value2 ["cnt" ]/ value ["cnt" ],2 ), key2 ))
47
- for key3 ,value3 in sorted (storage1 [key ]["messages" ][key2 ]["urls" ].items (), key = lambda x :x [1 ], reverse = True ):
48
- print ("[{}%] URL: {}" .format (round (100 * value3 / value2 ["cnt" ],2 ), key3 ))
50
+ print ("[{}] [{}%] Message: {}" .format (key , round (100 * value2 ["cnt" ]/ value ["cnt" ], 2 ), key2 ))
51
+ for key3 , value3 in sorted (storage1 [key ]["messages" ][key2 ]["urls" ].items (), key = lambda x : x [1 ], reverse = True ):
52
+ print ("[{}%] URL: {}" .format (round (100 * value3 / value2 ["cnt" ], 2 ), key3 ))
49
53
sec_end (key + key2 )
50
- for key2 ,value2 in sorted (storage2 [key ]["urls" ].items (), key = lambda x :x [1 ]['cnt' ], reverse = True ):
54
+ for key2 , value2 in sorted (storage2 [key ]["urls" ].items (), key = lambda x : x [1 ]['cnt' ], reverse = True ):
51
55
sec_start (key + key2 , key2 )
52
- print ("[{}] [{}%] URL: {}" .format (key , round (100 * value2 ["cnt" ]/ value ["cnt" ],2 ), key2 ))
53
- for key3 ,value3 in sorted (storage2 [key ]["urls" ][key2 ]["messages" ].items (), key = lambda x :x [1 ], reverse = True ):
54
- print ("[{}%] Message: {}" .format (round (100 * value3 / value2 ["cnt" ],2 ), key3 ))
56
+ print ("[{}] [{}%] URL: {}" .format (key , round (100 * value2 ["cnt" ]/ value ["cnt" ], 2 ), key2 ))
57
+ for key3 , value3 in sorted (storage2 [key ]["urls" ][key2 ]["messages" ].items (), key = lambda x : x [1 ], reverse = True ):
58
+ print ("[{}%] Message: {}" .format (round (100 * value3 / value2 ["cnt" ], 2 ), key3 ))
55
59
sec_end (key + key2 )
0 commit comments