|
| 1 | +import re |
| 2 | +import urllib.parse |
| 3 | + |
| 4 | +import requests |
1 | 5 | from bs4 import BeautifulSoup
|
| 6 | + |
2 | 7 | from uk_bin_collection.uk_bin_collection.common import *
|
3 | 8 | from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
|
4 | 9 |
|
5 |
| -from bs4 import BeautifulSoup |
6 |
| -import urllib.parse |
7 |
| -import requests |
8 |
| -import re |
9 |
| - |
10 | 10 |
|
11 | 11 | class CouncilClass(AbstractGetBinDataClass):
|
12 | 12 | def parse_data(self, page: str, **kwargs) -> dict:
|
13 | 13 |
|
14 | 14 | data = {"bins": []}
|
15 | 15 |
|
| 16 | + headers = { |
| 17 | + "Origin": "https://www.nuneatonandbedworth.gov.uk/", |
| 18 | + "Referer": "https://www.nuneatonandbedworth.gov.uk/", |
| 19 | + "User-Agent": "Mozilla/5.0", |
| 20 | + } |
| 21 | + |
16 | 22 | street = urllib.parse.quote_plus(kwargs.get("paon"))
|
17 | 23 | base_url = "https://www.nuneatonandbedworth.gov.uk/"
|
18 | 24 | search_query = f"directory/search?directoryID=3&showInMap=&keywords={street}&search=Search+directory"
|
19 | 25 |
|
20 |
| - search_response = requests.get(base_url + search_query) |
| 26 | + search_response = requests.get(base_url + search_query, headers=headers) |
21 | 27 |
|
22 | 28 | if search_response.status_code == 200:
|
23 | 29 | soup = BeautifulSoup(search_response.content, "html.parser")
|
@@ -56,7 +62,13 @@ def parse_data(self, page: str, **kwargs) -> dict:
|
56 | 62 |
|
57 | 63 | def get_bin_data(self, url) -> dict:
|
58 | 64 |
|
59 |
| - bin_day_response = requests.get(url) |
| 65 | + headers = { |
| 66 | + "Origin": "https://www.nuneatonandbedworth.gov.uk/", |
| 67 | + "Referer": "https://www.nuneatonandbedworth.gov.uk/", |
| 68 | + "User-Agent": "Mozilla/5.0", |
| 69 | + } |
| 70 | + |
| 71 | + bin_day_response = requests.get(url, headers=headers) |
60 | 72 |
|
61 | 73 | if bin_day_response.status_code == 200:
|
62 | 74 |
|
|
0 commit comments