[MV] Use data from WFS (#211)

Author: SimonMand

README.md

@@ -29,7 +29,7 @@ In details, the IDs are sourced as follows:
 |HB| `id` URL query param on the school's detail page (identical to the SNR (Schulnummer) from the overview page) | `HB-937`                                                                   |✅ likely|
 |HH| Field `schul_id` From the WFS Service                                                                        | `HH-7910-0`                                                                |✅ likely|
 |HE| `school_no` URL query param of the schools's details page (identical to the Dienststellennummer)             | `HE-4024`                                                                  |✅ likely|
-|MV| Column `DIENSTSTELLEN-NUMMER` from the XLSX file                                                             | `MV-75130302`                                                              |✅ likely|
+|MV| Field `dstnr` from the WFS                                                                                   | `MV-75130302`                                                              |✅ likely|
 |NI| Field `schulnr` from the JSON in the details payload                                                         | `NI-67763`                                                                 |✅ likely|
 |NW| Column `Schulnummer` from the CSV                                                                            | `NW-162437`                                                                |✅ likely|
 |RP| `Schulnummer` from the school's details page                                                                 | `RP-50720`                                                                 |✅ likely|
@@ -50,7 +50,7 @@ When available, we try to use the geolocations provided by the data publishers.
 | HB    | ❌ No                 | -                                            |
 | HH    | ✅ Yes                | WFS                                          |
 | HE    | ❌ No                 | -                                            |
-| MV    | ❌ No                 | -                                            |
+| MV    | ✅ Yes                | WFS                                          |
 | NI    | ❌ No                 | -                                            |
 | NW    | ✅ Yes                | Converted from EPSG:25832 in source CSV data |
 | RP    | ❌ No                 | -                                            |

jedeschule/spiders/bayern.py

@@ -27,7 +27,6 @@ def parse(self, response, **kwargs):
             members = [members]
 
         for member in members:
-            # Each member is a dict with one key = school tag, value = school data dict
             school = next(iter(member.values()), {})
 
             data_elem = {

jedeschule/spiders/mecklenburg_vorpommern.py

@@ -1,7 +1,5 @@
+import xmltodict
 from scrapy import Item
-from openpyxl import load_workbook
-from io import BytesIO
-
 
 from jedeschule.items import School
 from jedeschule.spiders.school_spider import SchoolSpider
@@ -16,40 +14,80 @@ def as_string(value: str):
 
 class MecklenburgVorpommernSpider(SchoolSpider):
     name = "mecklenburg-vorpommern"
-    # The state provides the data as an Excel file. The current year's
-    # data is for sale, all older version are free to download.
-    # We use the free data from 2022/2023
-    # An overview of all available files can be found here:
-    #   https://www.statistischebibliothek.de/mir/receive/MVSerie_mods_00000396
-    # Official documentation on all available data here:
-    #   https://www.laiv-mv.de/Statistik/Veröffentlichungen/Verzeichnisse/
-    base_url = "https://www.statistischebibliothek.de/mir/servlets/MCRFileNodeServlet/MVHeft_derivate_00007470/V044%202023%2000.xlsx"
-    start_urls = [base_url]
-
-    def parse(self, response):
-        workbook = load_workbook(filename=BytesIO(response.body), data_only=True)
-        data_sheet = workbook["Verzeichnis allg bild Schulen"]
-
-        rows = list(data_sheet.iter_rows(values_only=True))
-        headers = rows[0]
-
-        for row in rows[1:]:
-            yield {
-                headers[i]: row[i]
-                for i in range(len(headers))
-            }
+    start_urls = [
+        "https://www.geodaten-mv.de/dienste/schulstandorte_wfs?"
+        "SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&srsname=EPSG%3A4326&typeNames="
+        "ms:schultyp_grund,"
+        "ms:schultyp_regional,"
+        "ms:schultyp_gymnasium,"
+        "ms:schultyp_gesamt,"
+        "ms:schultyp_waldorf,"
+        "ms:schultyp_foerder,"
+        "ms:schultyp_abendgym,"
+        "ms:schultyp_berufs"
+    ]
+
+    def parse(self, response, **kwargs):
+        data = xmltodict.parse(response.text)
+
+        feature_collection = data.get("wfs:FeatureCollection", {})
+        members = feature_collection.get("wfs:member", [])
+
+        if not isinstance(members, list):
+            members = [members]
+
+        for member in members:
+            if "wfs:FeatureCollection" in member:
+                inner_members = member["wfs:FeatureCollection"].get("wfs:member", [])
+                if not isinstance(inner_members, list):
+                    inner_members = [inner_members]
+                
+                for inner_member in inner_members:
+                    school_data = next(iter(inner_member.values()), {})
+                    yield self._extract_school_data(school_data)
+            else:
+                school_data = next(iter(member.values()), {})
+                yield self._extract_school_data(school_data)
+
+    @staticmethod
+    def _extract_school_data(school):
+        data_elem = {}
+
+        for key, value in school.items():
+            if key == "ms:msGeometry":
+                point = value.get("gml:Point", {})
+                pos = point.get("gml:pos", "")
+                if pos:
+                    lat, lon = pos.split()
+                    data_elem["lat"] = float(lat)
+                    data_elem["lon"] = float(lon)
+            elif not key.startswith("@"):
+                clean_key = key.split(":", 1)[-1] if ":" in key else key
+                data_elem[clean_key] = value
+
+        return data_elem
 
     @staticmethod
     def normalize(item: Item) -> School:
+        def safe_strip(value):
+            if not value or not value.strip():
+                return None
+            return value.strip()
+        
         return School(
-            name=item.get("NAME1"),
-            id="MV-{}".format(as_string(item.get("DIENSTSTELLEN-NUMMER"))),
-            address=item.get("STRASSE"),
+            name=safe_strip(item.get("schulname")),
+            id="MV-{}".format(as_string(item.get("dstnr", ""))),
+            address=safe_strip(item.get("strassehnr")),
             address2="",
-            zip=as_string(item.get("PLZ")).zfill(5),
-            city=item.get("ORT"),
-            website=item.get("INTERNET"),
-            email=item.get("E-MAIL-ADRESSE"),
-            phone=item.get("TELEFON"),
-            director=item.get("SCHULLEITER/-IN"),
+            zip=as_string(item.get("plz", "")).zfill(5),
+            city=safe_strip(item.get("ort")),
+            website=safe_strip(item.get("internet")),
+            email=safe_strip(item.get("emailadresse")),
+            phone=safe_strip(item.get("telefon")),
+            director=safe_strip(item.get("schulleiter")),
+            school_type=safe_strip(item.get("orgform")),
+            legal_status=safe_strip(item.get("rechtsstatus")),
+            provider=safe_strip(item.get("schultraeger")),
+            latitude=item.get("lat"),
+            longitude=item.get("lon"),
         )

test/test_mecklenburg_vorpommern.py

@@ -0,0 +1,84 @@
+import unittest
+
+from scrapy.http import TextResponse
+
+from jedeschule.spiders.mecklenburg_vorpommern import MecklenburgVorpommernSpider
+
+
+class TestMecklenburgVorpommernSpider(unittest.TestCase):
+    def test_parse(self):
+        xml_response = """<?xml version='1.0' encoding="UTF-8" ?>
+<wfs:FeatureCollection
+   xmlns:ms="http://mapserver.gis.umn.edu/mapserver"
+   xmlns:gml="http://www.opengis.net/gml/3.2"
+   xmlns:wfs="http://www.opengis.net/wfs/2.0"
+   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+   xsi:schemaLocation="http://mapserver.gis.umn.edu/mapserver https://www.geodaten-mv.de/dienste/schulstandorte_wfs?SERVICE=WFS&amp;VERSION=2.0.0&amp;REQUEST=DescribeFeatureType&amp;TYPENAME=ms:schultyp_grund,ms:schultyp_regional,ms:schultyp_gymnasium,ms:schultyp_gesamt,ms:schultyp_waldorf,ms:schultyp_foerder,ms:schultyp_abendgym,ms:schultyp_berufs&amp;OUTPUTFORMAT=application%2Fgml%2Bxml%3B%20version%3D3.2 http://www.opengis.net/wfs/2.0 http://schemas.opengis.net/wfs/2.0/wfs.xsd http://www.opengis.net/gml/3.2 http://schemas.opengis.net/gml/3.2.1/gml.xsd"
+   timeStamp="2025-10-10T15:56:47" numberMatched="unknown" numberReturned="1"
+   next="https://www.geodaten-mv.de/dienste/schulstandorte_wfs?SERVICE=WFS&amp;REQUEST=GetFeature&amp;VERSION=2.0.0&amp;srsname=EPSG%3A4326&amp;typeNames=ms%3Aschultyp_grund%2Cms%3Aschultyp_regional%2Cms%3Aschultyp_gymnasium%2Cms%3Aschultyp_gesamt%2Cms%3Aschultyp_waldorf%2Cms%3Aschultyp_foerder%2Cms%3Aschultyp_abendgym%2Cms%3Aschultyp_berufs&amp;count=1&amp;STARTINDEX=1">
+    <wfs:boundedBy>
+        <gml:Envelope srsName="urn:ogc:def:crs:EPSG::4326">
+            <gml:lowerCorner>53.846900 11.977407</gml:lowerCorner>
+            <gml:upperCorner>53.846900 11.977407</gml:upperCorner>
+        </gml:Envelope>
+    </wfs:boundedBy>
+    <!-- WARNING: No featureid defined for typename 'schultyp_grund'. Output will not validate. -->
+    <wfs:member>
+        <ms:schultyp_grund>
+            <gml:boundedBy>
+                <gml:Envelope srsName="urn:ogc:def:crs:EPSG::4326">
+                    <gml:lowerCorner>53.846900 11.977407</gml:lowerCorner>
+                    <gml:upperCorner>53.846900 11.977407</gml:upperCorner>
+                </gml:Envelope>
+            </gml:boundedBy>
+            <ms:msGeometry>
+                <gml:Point gml:id=".1" srsName="urn:ogc:def:crs:EPSG::4326">
+                    <gml:pos>53.846900 11.977407</gml:pos>
+                </gml:Point>
+            </ms:msGeometry>
+            <ms:orgform>Grundschule</ms:orgform>
+            <ms:schultraeger></ms:schultraeger>
+            <ms:rechtsstatus>Öffentlich</ms:rechtsstatus>
+            <ms:schulname>Grundschule und Freizeithaus am Schloßplatz   </ms:schulname>
+            <ms:strassehnr>Schloßplatz 3</ms:strassehnr>
+            <ms:plz>18246</ms:plz>
+            <ms:ort>Bützow</ms:ort>
+            <ms:besonderheiten></ms:besonderheiten>
+            <ms:schulleiter>Frau Beuster</ms:schulleiter>
+            <ms:telefon>038461 - 52006</ms:telefon>
+            <ms:emailadresse>[email protected]</ms:emailadresse>
+            <ms:internet></ms:internet>
+            <ms:anzahl_klassen>12</ms:anzahl_klassen>
+            <ms:anzahl_schueler>247</ms:anzahl_schueler>
+            <ms:strasse>Schloßplatz</ms:strasse>
+            <ms:hnr>3</ms:hnr>
+            <ms:dstnr>75135304</ms:dstnr>
+        </ms:schultyp_grund>
+    </wfs:member>
+</wfs:FeatureCollection>"""
+
+        spider = MecklenburgVorpommernSpider()
+        response = TextResponse(url="https://test.com", body=xml_response, encoding="utf-8")
+        schools = list(spider.parse(response))
+        self.assertEqual(len(schools), 1)
+
+        school = schools[0]
+        parsed_school = spider.normalize(school)
+
+        self.assertEqual(parsed_school["id"], "MV-75135304")
+        self.assertEqual(parsed_school["name"], "Grundschule und Freizeithaus am Schloßplatz")
+        self.assertEqual(parsed_school["address"], "Schloßplatz 3")
+        self.assertEqual(parsed_school["city"], "Bützow")
+        self.assertEqual(parsed_school["school_type"], "Grundschule")
+        self.assertEqual(parsed_school["zip"], "18246")
+        self.assertEqual(parsed_school["latitude"], 53.846900)
+        self.assertEqual(parsed_school["longitude"], 11.977407)
+        self.assertEqual(parsed_school["legal_status"], "Öffentlich")
+        self.assertEqual(parsed_school["director"], "Frau Beuster")
+        self.assertEqual(parsed_school["phone"], "038461 - 52006")
+        self.assertEqual(parsed_school["email"], "[email protected]")
+        self.assertIsNone(parsed_school["website"])
+
+
+if __name__ == "__main__":
+    unittest.main()