Merge branch 'release_18_june' into fix-issue-1484

Author: robbrad

uk_bin_collection/tests/input.json

@@ -104,7 +104,7 @@
         "skip_get_url": true,
         "uprn": "766252532",
         "url": "http://avdcbins.web-labs.co.uk/RefuseApi.asmx",
-        "wiki_name": "Buckinghamshire (Aylesbury Vale)",
+        "wiki_name": "Buckinghamshire",
         "wiki_note": "To get the UPRN, please use [FindMyAddress](https://www.findmyaddress.co.uk/search). Returns all published collections in the past, present, future.",
         "LAD24CD": "E06000060"
     },
@@ -623,13 +623,14 @@
         "house_number": "2",
         "postcode": "CA13 0DE",
         "url": "https://www.allerdale.gov.uk",
-        "wiki_name": "Cumberland (Allerdale)",
+        "wiki_name": "Cumberland",
         "wiki_note": "Pass the house number and postcode in their respective parameters.",
         "LAD24CD": "E06000063"
     },
     "CumberlandCouncil": {
-        "uprn": "100110734613",
-        "url": "https://waste.cumberland.gov.uk",
+        "postcode": "CA6 5PH",
+        "uprn": "10009457328",
+        "url": "https://waste.cumberland.gov.uk/renderform?t=25&k=E43CEB1FB59F859833EF2D52B16F3F4EBE1CAB6A",
         "wiki_name": "Cumberland",
         "wiki_note": "Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.",
         "LAD24CD": "E06000063"
@@ -1360,7 +1361,7 @@
         "skip_get_url": true,
         "uprn": "12081498",
         "url": "https://www.ealing.gov.uk/site/custom_scripts/WasteCollectionWS/home/FindCollection",
-        "wiki_name": "Ealing (London Borough)",
+        "wiki_name": "Ealing",
         "wiki_note": "Pass the UPRN. You can find it using [FindMyAddress](https://www.findmyaddress.co.uk/search).",
         "LAD24CD": "E09000009"
     },
@@ -1828,8 +1829,8 @@
     "OxfordCityCouncil": {
         "postcode": "OX3 7QF",
         "uprn": "100120820551",
-        "url": "https://www.oxford.gov.uk",
-        "wiki_command_url_override": "https://www.oxford.gov.uk",
+        "url": "https://www.oxford.gov.uk/xfp/form/142",
+        "wiki_command_url_override": "https://www.oxford.gov.uk/xfp/form/142",
         "wiki_name": "Oxford",
         "wiki_note": "You will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN.",
         "LAD24CD": "E07000178"
@@ -2804,4 +2805,4 @@
         "wiki_note": "Provide your UPRN.",
         "LAD24CD": "E06000014"
     }
-}
+}

uk_bin_collection/uk_bin_collection/councils/CumberlandCouncil.py

@@ -16,6 +16,7 @@ class CouncilClass(AbstractGetBinDataClass):
     def parse_data(self, page: str, **kwargs) -> dict:
 
         user_uprn = kwargs.get("uprn")
+        postcode = kwargs.get("postcode")
         check_uprn(user_uprn)
         bindata = {"bins": []}
 
@@ -57,6 +58,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
             "TriggerCtl": "",
             "FF265": f"U{user_uprn}",
             "FF265lbltxt": "Please select your address",
+            "FF265-text": postcode
         }
 
         # print(payload)

uk_bin_collection/uk_bin_collection/councils/KingstonUponThamesCouncil.py

@@ -51,7 +51,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
                     if row.find("dt").get_text().strip().lower() == "next collection":
                         collection_date = remove_ordinal_indicator_from_date_string(
                             row.find("dd").get_text()
-                        ).strip()
+                        ).strip().replace(" (In progress)", "")
                         # strip out any text inside of the date string
                         collection_date = re.sub(
                             r"\n\s*\(this.*?\)", "", collection_date

uk_bin_collection/uk_bin_collection/councils/NorthHertfordshireDistrictCouncil.py

@@ -151,7 +151,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
 
             # Looking for bin types in the exact HTML structure
             bin_type_elements = soup.select(
-                "div.formatting_bold.formatting_size_bigger.formatting span.value-as-text"
+                "div.page_cell.contains_widget:first-of-type div.formatting_bold.formatting_size_bigger.formatting span.value-as-text"
             )
             # print(f"Found {len(bin_type_elements)} bin type elements")
 

uk_bin_collection/uk_bin_collection/councils/OxfordCityCouncil.py

@@ -21,8 +21,8 @@ def parse_data(self, page: str, **kwargs) -> dict:
         check_postcode(user_postcode)
         bindata = {"bins": []}
 
-        session_uri = "https://www.oxford.gov.uk/mybinday"
-        URI = "https://www.oxford.gov.uk/xfp/form/142"
+        session_uri = "https://www.oxford.gov.uk/xfp/form/142"
+        URI = "https://www.oxford.gov.uk/xfp/form/142#q6ad4e3bf432c83230a0347a6eea6c805c672efeb_0"
 
         session = requests.Session()
         token_response = session.get(session_uri)
@@ -40,15 +40,18 @@ def parse_data(self, page: str, **kwargs) -> dict:
 
         collection_response = session.post(URI, data=form_data)
 
-        collection_soup = BeautifulSoup(collection_response.text, "html.parser")
-        for paragraph in collection_soup.find("div", class_="editor").find_all("p"):
-            matches = re.match(r"^(\w+) Next Collection: (.*)", paragraph.text)
+        soup = BeautifulSoup(collection_response.text, "html.parser")
+        #print(soup)
+
+        for paragraph in soup.find("div", class_="editor").find_all("p"):
+            matches = re.match(r"^Your next (\w+) collections: (.*)", paragraph.text)
             if matches:
                 collection_type, date_string = matches.groups()
+                parts = date_string.split(', ', 1)
                 try:
-                    date = datetime.strptime(date_string, "%A %d %B %Y").date()
+                    date = datetime.strptime(parts[0], "%A %d %B %Y").date()
                 except ValueError:
-                    date = datetime.strptime(date_string, "%A %d %b %Y").date()
+                    date = datetime.strptime(parts[0], "%A %d %b %Y").date()
 
                 dict_data = {
                     "type": collection_type,
@@ -61,3 +64,5 @@ def parse_data(self, page: str, **kwargs) -> dict:
         )
 
         return bindata
+
+

uk_bin_collection/uk_bin_collection/councils/RugbyBoroughCouncil.py

@@ -113,7 +113,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
                             try:
                                 # Convert date from "Friday 09 May 2025" format
                                 parsed_date = datetime.strptime(
-                                    collection_date, "%A %d %B %Y"
+                                    collection_date, "%A %d %b %Y"
                                 )
                                 formatted_date = parsed_date.strftime("%d/%m/%Y")
 
@@ -132,7 +132,7 @@ def parse_data(self, page: str, **kwargs) -> dict:
                 key=lambda x: datetime.strptime(x["collectionDate"], "%d/%m/%Y")
             )
 
-            print(bin_data)
+            #print(bin_data)
 
         except Exception as e:
             # Here you can log the exception if needed

wiki/Councils.md

@@ -1158,9 +1158,10 @@ Note: Pass the house number and postcode in their respective parameters.
 
 ### Cumberland Borough Council
 ```commandline
-python collect_data.py CumberlandCouncil https://waste.cumberland.gov.uk -u XXXXXXXX
+python collect_data.py CumberlandCouncil https://waste.cumberland.gov.uk -u XXXXXXXX -p "XXXX XXX"
 ```
 Additional parameters:
+- `-p` - postcode
 - `-u` - UPRN
 
 Note: Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your UPRN.
@@ -2682,7 +2683,7 @@ Note: Replace UPRN in URL with your own UPRN.
 
 ### Oxford City Council
 ```commandline
-python collect_data.py OxfordCityCouncil https://www.oxford.gov.uk -u XXXXXXXX -p "XXXX XXX"
+python collect_data.py OxfordCityCouncil https://www.oxford.gov.uk/xfp/form/142 -u XXXXXXXX -p "XXXX XXX"
 ```
 Additional parameters:
 - `-u` - UPRN