Merge pull request #1602 from m26dvd/master

fix: Council Fix Pack - September 2025

Author: robbrad

uk_bin_collection/tests/input.json

@@ -102,12 +102,9 @@
     },
     "BCPCouncil": {
         "LAD24CD": "E06000058",
-        "house_number": "3 HARBOUR VIEW ROAD, POOLE, BH14 0PD",
-        "postcode": "BH14 0PD",
-        "web_driver": "http://selenium:4444",
         "skip_get_url": true,
         "uprn": "100040810214",
-        "url": "https://online.bcpcouncil.gov.uk/bindaylookup/",
+        "url": "https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection",
         "wiki_name": "Bournemouth, Christchurch and Poole",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN."
     },
@@ -888,7 +885,7 @@
     "ErewashBoroughCouncil": {
         "skip_get_url": true,
         "uprn": "10003582028",
-        "url": "https://map.erewash.gov.uk/isharelive.web/myerewash.aspx",
+        "url": "https://www.erewash.gov.uk",
         "wiki_name": "Erewash",
         "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E07000036"
@@ -995,7 +992,7 @@
     },
     "GlasgowCityCouncil": {
         "uprn": "906700034497",
-        "url": "https://onlineservices.glasgow.gov.uk/forms/RefuseAndRecyclingWebApplication/AddressSearch.aspx",
+        "url": "https://onlineservices.glasgow.gov.uk/forms/refuseandrecyclingcalendar/AddressSearch.aspx",
         "skip_get_url": true,
         "wiki_name": "Glasgow City",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",

uk_bin_collection/uk_bin_collection/councils/BCPCouncil.py

@@ -1,15 +1,13 @@
-import json
 import time
-from datetime import datetime
-from bs4 import BeautifulSoup
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait, Select
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.common.keys import Keys
+
+import requests
+from dateutil.relativedelta import relativedelta
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
 
+# import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """
     Concrete classes have to implement all abstract operations of the
@@ -18,116 +16,43 @@ class CouncilClass(AbstractGetBinDataClass):
     """
 
     def parse_data(self, page: str, **kwargs) -> dict:
-        postcode = kwargs.get("postcode")
-        house_number = kwargs.get("paon")
-        web_driver = kwargs.get("web_driver")
-        headless = kwargs.get("headless", True)
-        
-        check_postcode(postcode)
-        check_paon(house_number)
-        
-        driver = create_webdriver(web_driver, headless=headless)
-        
-        try:
-            driver.get("https://bcpportal.bcpcouncil.gov.uk/checkyourbincollection/")
-            
-            # Handle cookie banner first
-            try:
-                cookie_button = WebDriverWait(driver, 5).until(
-                    EC.element_to_be_clickable((By.XPATH, "//button[contains(text(), 'Okay')]"))
-                )
-                cookie_button.click()
-            except:
-                pass  # Cookie banner might not be present
-            
-            # Wait for and enter postcode
-            postcode_input = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.CSS_SELECTOR, "input[type='text']"))
-            )
-            postcode_input.clear()
-            postcode_input.send_keys(postcode)
-            
-            # Click the search span element
-            search_button = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.ID, "searchAddress"))
-            )
-            search_button.click()
-            
-            # Wait for address dropdown
-            select_element = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.TAG_NAME, "select"))
-            )
-            
-            # Find and select the address containing the house number
-            address_option = WebDriverWait(driver, 10).until(
-                EC.element_to_be_clickable((By.XPATH, f"//option[contains(text(), 'HARBOUR VIEW ROAD')]"))
-            )
-            address_option.click()
-            
-            # Wait for bin collection results to load
-            WebDriverWait(driver, 15).until(
-                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')] | //th[contains(text(), 'collection')]"))
-            )
-            
-            # Find the table containing collection data by looking for a cell with 'collection' text
-            collection_table = WebDriverWait(driver, 10).until(
-                EC.presence_of_element_located((By.XPATH, "//td[contains(text(), 'collection')]/ancestor::table | //th[contains(text(), 'collection')]/ancestor::table"))
-            )
-            
-            # Parse the table data
-            soup = BeautifulSoup(driver.page_source, 'html.parser')
-            data = {"bins": []}
-            
-            # Find the table containing collection information
-            collection_cell = soup.find(['td', 'th'], string=lambda text: text and 'collection' in text.lower())
-            if collection_cell:
-                table = collection_cell.find_parent('table')
-                if table:
-                    rows = table.find_all('tr')
-                    for row in rows[1:]:  # Skip header row
-                        cells = row.find_all(['td', 'th'])
-                        if len(cells) >= 2:  # At least bin type and one collection date
-                            bin_type = cells[0].get_text(strip=True)
-                            next_collection = cells[1].get_text(strip=True) if len(cells) > 1 else ""
-                            following_collection = cells[2].get_text(strip=True) if len(cells) > 2 else ""
-                            
-                            
-                            # Process next collection date
-                            if bin_type and next_collection and "No collection" not in next_collection:
-                                try:
-                                    # Try multiple date formats
-                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
-                                        try:
-                                            parsed_date = datetime.strptime(next_collection, date_fmt)
-                                            data["bins"].append({
-                                                "type": bin_type,
-                                                "collectionDate": parsed_date.strftime(date_format)
-                                            })
-                                            break
-                                        except ValueError:
-                                            continue
-                                except:
-                                    continue
-                            
-                            # Process following collection date
-                            if bin_type and following_collection and "No collection" not in following_collection and "download PDF" not in following_collection:
-                                try:
-                                    # Clean up the following collection text (remove PDF link text)
-                                    following_collection = following_collection.replace("download PDF", "").strip()
-                                    for date_fmt in ["%A, %d %B %Y", "%A %d %B %Y", "%d/%m/%Y", "%d-%m-%Y", "%Y-%m-%d"]:
-                                        try:
-                                            parsed_date = datetime.strptime(following_collection, date_fmt)
-                                            data["bins"].append({
-                                                "type": bin_type,
-                                                "collectionDate": parsed_date.strftime(date_format)
-                                            })
-                                            break
-                                        except ValueError:
-                                            continue
-                                except:
-                                    continue
-            
-            return data
-            
-        finally:
-            driver.quit()
+        # Make a BS4 object
+        uprn = kwargs.get("uprn")
+        # usrn = kwargs.get("paon")
+        check_uprn(uprn)
+        # check_usrn(usrn)
+        bindata = {"bins": []}
+
+        # uprn = uprn.zfill(12)
+
+        API_URL = "https://prod-17.uksouth.logic.azure.com/workflows/58253d7b7d754447acf9fe5fcf76f493/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=TAvYIUFj6dzaP90XQCm2ElY6Cd34ze05I3ba7LKTiBs"
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "*/*",
+            "User-Agent": "Mozilla/5.0",
+            "Referer": "https://bcpportal.bcpcouncil.gov.uk/",
+        }
+        s = requests.session()
+        data = {
+            "uprn": uprn,
+        }
+
+        r = s.post(API_URL, json=data, headers=headers)
+        r.raise_for_status()
+
+        data = r.json()
+        rows_data = data["data"]
+        for row in rows_data:
+            bin_type = row["wasteContainerUsageTypeDescription"]
+            collections = row["scheduleDateRange"]
+            for collection in collections:
+                dict_data = {
+                    "type": bin_type,
+                    "collectionDate": datetime.strptime(
+                        collection, "%Y-%m-%d"
+                    ).strftime(date_format),
+                }
+                bindata["bins"].append(dict_data)
+
+        return bindata

uk_bin_collection/uk_bin_collection/councils/BasingstokeCouncil.py

@@ -1,6 +1,8 @@
-from bs4 import BeautifulSoup
 from datetime import datetime
+
 import requests
+from bs4 import BeautifulSoup
+
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
@@ -10,6 +12,7 @@
     "glass": "rteelem_ctl03_pnlCollections_Glass",
     # Garden waste data is only returned if the property is subscribed to the Garden Waste service
     "garden": "rteelem_ctl03_pnlCollections_GardenWaste",
+    "food": "rteelem_ctl03_pnlCollections_Food",
 }
 
 

uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py

@@ -72,7 +72,9 @@ def parse_data(self, page: str, **kwargs) -> dict:
                     break
 
             if not found:
-                raise Exception(f"Address containing '{user_paon}' not found in dropdown options")
+                raise Exception(
+                    f"Address containing '{user_paon}' not found in dropdown options"
+                )
 
             submit_btn = wait.until(
                 EC.presence_of_element_located(
@@ -84,7 +86,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
 
             results = wait.until(
                 EC.presence_of_element_located(
-                    (By.XPATH, f'//span[contains(@class,"collection-sub")]')
+                    (By.XPATH, f'//div[contains(@class,"mx-name-listView1")]')
                 )
             )
 
@@ -96,44 +98,21 @@ def parse_data(self, page: str, **kwargs) -> dict:
             current_date = datetime.now()
 
             # Find all elements with class starting with 'mx-name-index-'
-            bins = soup.find_all(class_=lambda x: x and x.startswith("mx-name-index-"))
+            bin_view = soup.find(class_="mx-name-listView1")
+            bins = bin_view.find_all(
+                class_=lambda x: x and x.startswith("mx-name-index-")
+            )
 
             for bin_item in bins:
-                bin_type = bin_item.find(class_="collection-main").text.strip()
-                day_of_week_elements = bin_item.find_all(class_="collection-header")
-                bin_date = None
-
-                for elem in day_of_week_elements:
-                    if (
-                        elem.text.strip() != bin_type
-                    ):  # Avoid taking the bin type as the date
-                        next_sibling = elem.find_next_sibling()
-                        if next_sibling:
-                            bin_date_str = next_sibling.text.strip()
-                            try:
-                                # Try parsing the date string in the format 'dd Month' (e.g., '30 Dec', '5 January')
-                                bin_date = datetime.strptime(bin_date_str, "%d %b")
-                            except ValueError:
-                                try:
-                                    # If the above format fails, try 'dd MonthName' (e.g., '30 December', '5 January')
-                                    bin_date = datetime.strptime(bin_date_str, "%d %B")
-                                except ValueError:
-                                    pass
-
-                            if bin_date:
-                                # Set the year based on the logic provided
-                                if bin_date.month < current_date.month:
-                                    bin_date = bin_date.replace(
-                                        year=current_date.year + 1
-                                    )
-                                else:
-                                    bin_date = bin_date.replace(year=current_date.year)
-                                # Format the date to the desired format
-                                bin_date = bin_date.strftime("%d/%m/%Y")
-                                break
+                bin_type = bin_item.find(class_="mx-name-text31").text.strip()
+
+                bin_date_str = bin_item.find(class_="mx-name-text29").text.strip()
+
+                bin_date = datetime.strptime(bin_date_str, "%d %B %Y")
+                bin_date = bin_date.strftime(date_format)
+
                 dict_data = {"type": bin_type, "collectionDate": bin_date}
                 data["bins"].append(dict_data)
-                print(data)
         except Exception as e:
             # Here you can log the exception if needed
             print(f"An error occurred: {e}")

uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py

@@ -26,7 +26,9 @@ def parse_data(self, page: str, **kwargs) -> dict:
         uprn = kwargs.get("uprn")
         check_uprn(uprn)
 
-        post_url = "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=myStreet.displayDetails"
+        base_url = "https://apps.castlepoint.gov.uk/cpapps/"
+
+        post_url = f"{base_url}index.cfm?fa=myStreet.displayDetails"
         post_header_str = (
             "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
             "image/apng,"
@@ -51,31 +53,60 @@ def parse_data(self, page: str, **kwargs) -> dict:
         soup = BeautifulSoup(post_response.text, features="html.parser")
         soup.prettify()
 
+        calMonthNext = f"{base_url}{soup.select_one("div.calMonthNext a")["href"]}"
+        nextmonth_response = requests.post(
+            calMonthNext, headers=post_headers, data=form_data, verify=False
+        )
+        soup_nextmonth = BeautifulSoup(nextmonth_response.text, features="html.parser")
+        soup_nextmonth.prettify()
+
         data = {"bins": []}
-        collection_tuple = []
 
-        calendar = soup.find("table", class_="calendar")
-        month = datetime.strptime(
-            soup.find("div", class_="calMonthCurrent").get_text(), "[%b]"
-        ).strftime("%m")
-        year = datetime.strptime(
-            soup.find("h1").get_text(), "About my Street - %B %Y"
-        ).strftime("%Y")
-
-        pink_days = [
-            day.get_text().strip() for day in calendar.find_all("td", class_="pink")
-        ]
-        black_days = [
-            day.get_text().strip() for day in calendar.find_all("td", class_="normal")
-        ]
-
-        for day in pink_days:
-            collection_date = datetime(year=int(year), month=int(month), day=int(day))
-            collection_tuple.append(("Pink collection", collection_date))
-
-        for day in black_days:
-            collection_date = datetime(year=int(year), month=int(month), day=int(day))
-            collection_tuple.append(("Normal collection", collection_date))
+        def parse_calendar_month(soup_one_month):
+            out = []
+
+            calendar = soup_one_month.find("table", class_="calendar")
+            if not calendar:
+                return out  # be robust
+
+            # e.g. "[Aug]"
+            month_txt = soup_one_month.find("div", class_="calMonthCurrent").get_text(
+                strip=True
+            )
+            month = datetime.strptime(month_txt, "[%b]").strftime("%m")
+
+            # e.g. "About my Street - August 2025"
+            year_txt = soup_one_month.find("h1").get_text(strip=True)
+            year = datetime.strptime(year_txt, "About my Street - %B %Y").strftime("%Y")
+
+            pink_days = [
+                td.get_text(strip=True) for td in calendar.find_all("td", class_="pink")
+            ]
+            black_days = [
+                td.get_text(strip=True)
+                for td in calendar.find_all("td", class_="normal")
+            ]
+
+            for day in pink_days:
+                out.append(
+                    (
+                        "Pink collection",
+                        datetime(year=int(year), month=int(month), day=int(day)),
+                    )
+                )
+            for day in black_days:
+                out.append(
+                    (
+                        "Normal collection",
+                        datetime(year=int(year), month=int(month), day=int(day)),
+                    )
+                )
+
+            return out
+
+        collection_tuple = []
+        for s in (soup, soup_nextmonth):
+            collection_tuple.extend(parse_calendar_month(s))
 
         ordered_data = sorted(collection_tuple, key=lambda x: x[1])