fix: #1571 - Castle Point District Council

fix: #1571 - Castle Point District Council

Author: m26dvd

uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py

@@ -26,7 +26,9 @@ def parse_data(self, page: str, **kwargs) -> dict:
         uprn = kwargs.get("uprn")
         check_uprn(uprn)
 
-        post_url = "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=myStreet.displayDetails"
+        base_url = "https://apps.castlepoint.gov.uk/cpapps/"
+
+        post_url = f"{base_url}index.cfm?fa=myStreet.displayDetails"
         post_header_str = (
             "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
             "image/apng,"
@@ -51,31 +53,60 @@ def parse_data(self, page: str, **kwargs) -> dict:
         soup = BeautifulSoup(post_response.text, features="html.parser")
         soup.prettify()
 
+        calMonthNext = f"{base_url}{soup.select_one("div.calMonthNext a")["href"]}"
+        nextmonth_response = requests.post(
+            calMonthNext, headers=post_headers, data=form_data, verify=False
+        )
+        soup_nextmonth = BeautifulSoup(nextmonth_response.text, features="html.parser")
+        soup_nextmonth.prettify()
+
         data = {"bins": []}
-        collection_tuple = []
 
-        calendar = soup.find("table", class_="calendar")
-        month = datetime.strptime(
-            soup.find("div", class_="calMonthCurrent").get_text(), "[%b]"
-        ).strftime("%m")
-        year = datetime.strptime(
-            soup.find("h1").get_text(), "About my Street - %B %Y"
-        ).strftime("%Y")
-
-        pink_days = [
-            day.get_text().strip() for day in calendar.find_all("td", class_="pink")
-        ]
-        black_days = [
-            day.get_text().strip() for day in calendar.find_all("td", class_="normal")
-        ]
-
-        for day in pink_days:
-            collection_date = datetime(year=int(year), month=int(month), day=int(day))
-            collection_tuple.append(("Pink collection", collection_date))
-
-        for day in black_days:
-            collection_date = datetime(year=int(year), month=int(month), day=int(day))
-            collection_tuple.append(("Normal collection", collection_date))
+        def parse_calendar_month(soup_one_month):
+            out = []
+
+            calendar = soup_one_month.find("table", class_="calendar")
+            if not calendar:
+                return out  # be robust
+
+            # e.g. "[Aug]"
+            month_txt = soup_one_month.find("div", class_="calMonthCurrent").get_text(
+                strip=True
+            )
+            month = datetime.strptime(month_txt, "[%b]").strftime("%m")
+
+            # e.g. "About my Street - August 2025"
+            year_txt = soup_one_month.find("h1").get_text(strip=True)
+            year = datetime.strptime(year_txt, "About my Street - %B %Y").strftime("%Y")
+
+            pink_days = [
+                td.get_text(strip=True) for td in calendar.find_all("td", class_="pink")
+            ]
+            black_days = [
+                td.get_text(strip=True)
+                for td in calendar.find_all("td", class_="normal")
+            ]
+
+            for day in pink_days:
+                out.append(
+                    (
+                        "Pink collection",
+                        datetime(year=int(year), month=int(month), day=int(day)),
+                    )
+                )
+            for day in black_days:
+                out.append(
+                    (
+                        "Normal collection",
+                        datetime(year=int(year), month=int(month), day=int(day)),
+                    )
+                )
+
+            return out
+
+        collection_tuple = []
+        for s in (soup, soup_nextmonth):
+            collection_tuple.extend(parse_calendar_month(s))
 
         ordered_data = sorted(collection_tuple, key=lambda x: x[1])