-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathnew_nepali_data.py
More file actions
154 lines (133 loc) · 6.18 KB
/
new_nepali_data.py
File metadata and controls
154 lines (133 loc) · 6.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
from lxml import etree
import re
import html
import requests
import unicodecsv
import csv
import sys
import codecs
import io
fullfile = [['mfid', 'incident_year', 'computer_id', 'incident_district', 'incident_vdc', 'incident_ward', 'incident_toll', 'origin_district', 'origin_vdc', 'origin_ward', 'origin_toll']]
for person_id in range(0, 10802):
current_record = []
# Requesting the data looping over several person IDs
url = "http://www.insec.org.np/victim/candidate_details_user.php?MFID=" + str(person_id)
response = requests.request("GET", url)
response.encoding = "UTF-8"
responsestring = response.text
print(person_id)
if len(responsestring)>20000:
# MFID:
current_record.append(str(person_id))
# Incident Year:
search_pattern = re.compile(r'(साल:\s)<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
current_record.append(wanteddata.strip())
else: current_record.append('NA')
# Computer ID:
search_pattern = re.compile(r'(कम्प्युटर अ\. नं\.\n )<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
current_record.append(wanteddata.strip())
else: current_record.append('NA')
# District of Incidence:
search_pattern = re.compile(r'(घटना भएको जिल्ला:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
if "&#" not in wanteddata:
current_record.append(wanteddata.strip())
else:
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Incidet VDC:
search_pattern = re.compile(r'(<td width="214">गाविस:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
if "&#" not in wanteddata:
current_record.append(wanteddata.strip())
else:
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Incident Ward:
search_pattern = re.compile(r'(<td width="135">वार्ड नं\.:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Incidet Toll:
search_pattern = re.compile(r'(<td>टोलको नाम:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
if "&#" not in wanteddata:
current_record.append(wanteddata.strip())
else:
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Origin District:
search_pattern = re.compile(r'(<td width="191"><strong> </strong>जिल्ला:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
if "&#" not in wanteddata:
current_record.append(wanteddata.strip())
else:
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Origin VDC:
search_pattern = re.compile(r'(<td colspan="2">गाविस:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
if "&#" not in wanteddata:
current_record.append(wanteddata.strip())
else:
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Origin Ward:
search_pattern = re.compile(r'(<td>वार्ड नं\.:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
# Origin Toll:
search_pattern = re.compile(r'(<td colspan="2">टोल:)\s<i>(.+)</i>')
founddata = search_pattern.search(responsestring)
if founddata != None:
wanteddata = founddata.groups()[1]
if "&#" not in wanteddata:
current_record.append(wanteddata.strip())
else:
current_record.append(html.unescape(wanteddata.strip()))
else: current_record.append('NA')
fullfile.append(current_record)
for line in fullfile:
stringline = ';'.join(line)
print(stringline)
##encodedfile = []
##for line in fullfile:
## encodedline = [x.encode('UTF8') for x in line]
## encodedfile.append(encodedline)
##print(encodedfile)
##
##with codecs.open('nepali_data.csv', 'a') as myfile:
## wr = csv.writer(myfile)
## for line in fullfile:
## stringline = ','.join(line)
## wr.writerow(stringline)
##
##with io.open('nepali_data.csv', 'w', encoding='iso-8859-1') as myfile:
## for line in fullfile:
## stringline = ','.join(line)
## myfile.write(stringline)
##
##with open('nepali_data.csv', 'wb') as myfile:
## wr = unicodecsv.writer(myfile, encoding='cp1252')
## wr.writerows(fullfile)