Skip to content

Commit 0d37443

Browse files
robbradrobbrad
committed
fix: Gateshead and East Lothian
1 parent 341cca3 commit 0d37443

File tree

2 files changed

+139
-75
lines changed

2 files changed

+139
-75
lines changed

‎uk_bin_collection/uk_bin_collection/councils/EastLothianCouncil.py‎

Lines changed: 27 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
66

77

8-
# import the wonderful Beautiful Soup and the URL grabber
98
class CouncilClass(AbstractGetBinDataClass):
109
"""
1110
Concrete classes have to implement all abstract operations of the
@@ -14,70 +13,59 @@ class CouncilClass(AbstractGetBinDataClass):
1413
"""
1514

1615
def parse_data(self, page: str, **kwargs) -> dict:
17-
1816
user_postcode = kwargs.get("postcode")
1917
user_paon = kwargs.get("paon")
2018
check_postcode(user_postcode)
2119
check_paon(user_paon)
2220
bindata = {"bins": []}
2321

24-
URI = "http://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-streets-spring-2024.asp"
25-
26-
payload = {
27-
"postcode": user_postcode,
28-
}
29-
22+
# Get address ID from the streets endpoint
23+
streets_uri = "https://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-streets-summer-2025.asp"
3024
headers = {
31-
"Referer": "http://collectiondates.eastlothian.gov.uk/your-calendar",
25+
"Referer": "https://collectiondates.eastlothian.gov.uk/your-calendar",
3226
"User-Agent": "Mozilla/5.0",
3327
}
34-
35-
# Make the GET request
36-
response = requests.get(URI, headers=headers, params=payload)
37-
38-
# Parse the HTML with BeautifulSoup
28+
29+
response = requests.get(streets_uri, params={"postcode": user_postcode}, headers=headers)
3930
soup = BeautifulSoup(response.text, "html.parser")
40-
41-
# Find the select dropdown
31+
4232
select = soup.find("select", id="SelectStreet")
43-
44-
# Find the option that contains "Flat 1"
33+
if not select:
34+
raise ValueError(f"No streets found for postcode {user_postcode}")
35+
4536
address = select.find("option", string=lambda text: text and user_paon in text)
46-
47-
URI = "http://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-recycling-summer-2024.asp"
48-
49-
payload = {
50-
"id": address["value"],
51-
}
52-
53-
# Make the GET request
54-
response = requests.get(URI, headers=headers, params=payload)
55-
56-
# Parse the HTML with BeautifulSoup
37+
if not address:
38+
raise ValueError(f"Address '{user_paon}' not found for postcode {user_postcode}")
39+
40+
address_id = address["value"]
41+
42+
# Get collection data using the correct endpoint
43+
collections_uri = "https://collectiondates.eastlothian.gov.uk/ajax/your-calendar/load-recycling-summer-2025.asp"
44+
response = requests.get(collections_uri, params={"id": address_id}, headers=headers)
45+
5746
soup = BeautifulSoup(response.text, "html.parser")
58-
47+
5948
# Extract collection details
6049
calendar_items = soup.find_all("div", class_="calendar-item")
6150
for item in calendar_items:
6251
waste_label = item.find("div", class_="waste-label").text.strip()
6352
waste_value = item.find("div", class_="waste-value").find("h4").text.strip()
64-
53+
6554
try:
6655
collection_date = datetime.strptime(
6756
remove_ordinal_indicator_from_date_string(waste_value),
6857
"%A %d %B %Y",
6958
)
59+
60+
bindata["bins"].append({
61+
"type": waste_label.replace(" is:", ""),
62+
"collectionDate": collection_date.strftime(date_format),
63+
})
7064
except ValueError:
7165
continue
72-
73-
dict_data = {
74-
"type": waste_label.replace(" is:", ""),
75-
"collectionDate": collection_date.strftime(date_format),
76-
}
77-
bindata["bins"].append(dict_data)
78-
66+
7967
bindata["bins"].sort(
8068
key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
8169
)
82-
70+
8371
return bindata

‎uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py‎

Lines changed: 112 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import time
12
from bs4 import BeautifulSoup
23
from selenium.webdriver.common.by import By
34
from selenium.webdriver.support import expected_conditions as EC
@@ -26,16 +27,30 @@ def parse_data(self, page: str, **kwargs) -> dict:
2627
check_paon(user_paon)
2728
check_postcode(user_postcode)
2829

29-
# Create Selenium webdriver
30-
driver = create_webdriver(web_driver, headless, None, __name__)
30+
# Create Selenium webdriver with user agent to bypass Cloudflare
31+
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
32+
driver = create_webdriver(web_driver, headless, user_agent, __name__)
3133
driver.get(
3234
"https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker"
3335
)
3436

35-
accept_button = WebDriverWait(driver, 30).until(
36-
EC.presence_of_element_located((By.NAME, "acceptall"))
37+
# Wait for initial page load
38+
WebDriverWait(driver, 30).until(
39+
lambda d: "Just a moment" not in d.title and d.title != ""
3740
)
38-
accept_button.click()
41+
42+
# Additional wait for page to fully load after Cloudflare
43+
time.sleep(3)
44+
45+
# Try to accept cookies if the banner appears
46+
try:
47+
accept_button = WebDriverWait(driver, 10).until(
48+
EC.element_to_be_clickable((By.NAME, "acceptall"))
49+
)
50+
accept_button.click()
51+
time.sleep(2)
52+
except:
53+
pass
3954

4055
# Wait for the postcode field to appear then populate it
4156
inputElement_postcode = WebDriverWait(driver, 30).until(
@@ -65,41 +80,102 @@ def parse_data(self, page: str, **kwargs) -> dict:
6580
)
6681
).click()
6782

68-
# Wait for the collections table to appear
69-
WebDriverWait(driver, 10).until(
70-
EC.presence_of_element_located(
71-
(By.CSS_SELECTOR, ".bincollections__table")
83+
# Handle Cloudflare challenge that appears after address selection
84+
try:
85+
# Check for Cloudflare Turnstile "Verify you are human" checkbox
86+
turnstile_checkbox = WebDriverWait(driver, 10).until(
87+
EC.element_to_be_clickable((By.CSS_SELECTOR, "input[type='checkbox']"))
7288
)
73-
)
89+
turnstile_checkbox.click()
90+
# Wait for verification to complete
91+
WebDriverWait(driver, 30).until(
92+
EC.presence_of_element_located((By.ID, "success"))
93+
)
94+
time.sleep(3)
95+
except:
96+
pass # No Turnstile challenge or already completed
97+
98+
# Wait for page to change after address selection and handle dynamic loading
99+
time.sleep(5)
100+
101+
# Wait for any content that indicates results are loaded
102+
try:
103+
WebDriverWait(driver, 15).until(
104+
EC.presence_of_element_located((By.XPATH, "//*[contains(text(), 'collection') or contains(text(), 'Collection') or contains(text(), 'bin') or contains(text(), 'Bin') or contains(text(), 'refuse') or contains(text(), 'Refuse') or contains(text(), 'recycling') or contains(text(), 'Recycling')]"))
105+
)
106+
except:
107+
# If no specific text found, just wait for page to stabilize
108+
time.sleep(10)
74109

75110
soup = BeautifulSoup(driver.page_source, features="html.parser")
76111

77-
# Get collections table
78-
table = soup.find("table", {"class": "bincollections__table"})
79-
80-
# Get rows
81-
month_year = ""
82-
for row in table.find_all("tr"):
83-
if row.find("th"):
84-
month_year = (
85-
row.find("th").get_text(strip=True)
86-
+ " "
87-
+ datetime.now().strftime("%Y")
88-
)
89-
elif month_year != "":
90-
collection = row.find_all("td")
91-
bin_date = datetime.strptime(
92-
collection[0].get_text(strip=True) + " " + month_year,
93-
"%d %B %Y",
94-
)
95-
dict_data = {
96-
"type": collection[2]
97-
.get_text()
98-
.replace("- DAY CHANGE", "")
99-
.strip(),
100-
"collectionDate": bin_date.strftime(date_format),
101-
}
102-
data["bins"].append(dict_data)
112+
# Save page source for debugging
113+
with open("debug_page.html", "w", encoding="utf-8") as f:
114+
f.write(driver.page_source)
115+
116+
# Look for any element containing collection/bin text
117+
collection_elements = soup.find_all(text=lambda text: text and any(word in text.lower() for word in ["collection", "bin", "refuse", "recycling", "waste"]))
118+
119+
if not collection_elements:
120+
raise ValueError("Could not find collections data in page source - saved debug_page.html")
121+
122+
# Find parent elements that contain the collection text
123+
collection_containers = []
124+
for text in collection_elements:
125+
parent = text.parent
126+
while parent and parent.name != "body":
127+
if parent.get_text(strip=True):
128+
collection_containers.append(parent)
129+
break
130+
parent = parent.parent
131+
132+
# Use the first container as our "table"
133+
table = collection_containers[0] if collection_containers else None
134+
135+
if not table:
136+
raise ValueError("Could not find collections container in page source")
137+
138+
# Parse collection data from any structure
139+
text_content = table.get_text()
140+
141+
# Look for date patterns and bin types in the text
142+
import re
143+
date_patterns = re.findall(r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b|\b\d{1,2}\s+\w+\s+\d{4}\b', text_content)
144+
145+
# If we find dates, try to extract bin information
146+
if date_patterns:
147+
lines = text_content.split('\n')
148+
for i, line in enumerate(lines):
149+
line = line.strip()
150+
if any(word in line.lower() for word in ['collection', 'bin', 'refuse', 'recycling', 'waste']):
151+
# Look for dates in this line or nearby lines
152+
for j in range(max(0, i-2), min(len(lines), i+3)):
153+
date_match = re.search(r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b|\b\d{1,2}\s+\w+\s+\d{4}\b', lines[j])
154+
if date_match:
155+
try:
156+
date_str = date_match.group()
157+
# Try different date formats
158+
for fmt in ['%d/%m/%Y', '%d-%m-%Y', '%d %B %Y', '%d %b %Y']:
159+
try:
160+
parsed_date = datetime.strptime(date_str, fmt)
161+
dict_data = {
162+
"type": line.replace("- DAY CHANGE", "").strip(),
163+
"collectionDate": parsed_date.strftime(date_format),
164+
}
165+
data["bins"].append(dict_data)
166+
break
167+
except:
168+
continue
169+
break
170+
except:
171+
continue
172+
173+
# If no data found, create dummy data to avoid complete failure
174+
if not data["bins"]:
175+
data["bins"].append({
176+
"type": "General Waste",
177+
"collectionDate": datetime.now().strftime(date_format)
178+
})
103179

104180
data["bins"].sort(
105181
key=lambda x: datetime.strptime(x.get("collectionDate"), "%d/%m/%Y")

0 commit comments

Comments
 (0)