Merge pull request #1581 from IFRCGo/develop

Hotfix for DREF MDR Code validation fix

Author: szabozoltan69

CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## 1.1.460
+
+### Added
+ - Fix for MDR code validation
+ - Admin2 enhancements
+
 ## 1.1.459
 
 ### Added
@@ -2084,7 +2090,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## 0.1.20
 
-[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.459...HEAD
+[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.460...HEAD
+[1.1.460]: https://github.com/IFRCGo/go-api/compare/1.1.459...1.1.460
 [1.1.459]: https://github.com/IFRCGo/go-api/compare/1.1.458...1.1.459
 [1.1.458]: https://github.com/IFRCGo/go-api/compare/1.1.457...1.1.458
 [1.1.457]: https://github.com/IFRCGo/go-api/compare/1.1.456...1.1.457

README.md

@@ -215,6 +215,24 @@ The above command will generate a list of missing districts in the database base
 * `--import-missing missing-districts.txt` -- this will import districts for the iso2 mentioned in `missing-districts.txt` to the database. The file is the same format as generated by the default command.
 * `--import-all` -- this option is used to import all districts in the shapefile, if they don't have a code we can match against in the database.
 
+## import-admin2-data
+This management command is used for updating and importing admin2 shapefile. To run:
+* `python manage.py import-admin2-data <filename.shp>`
+
+The shapefile should have the following mandatory fields:
+* name or shapeName
+* code or pcode
+* admin1_id (this is the ID of the GO district this admin2 belongs to)
+
+See [this ticket](https://github.com/IFRCGo/go-api/issues/1492#issuecomment-1284120696) for a full workflow of preparing the admin2 shapefiles.
+The above command will generate a list of missing admin2s in the database based on the code (we use pcodes) to a file called `missing-admin2.txt`
+
+### Options available for the command
+* `--update-geom` -- updates the geometry for all admin2 matched in the shapefile.
+* `--import-missing missing-admin2.txt` -- this will import admin2 listed in `missing-admin2.txt` to the database. The file is the same format as generated by the default command.
+* `--import-all` -- this option is used to import all admin2 in the shapefile.
+
+
 ## Update bbox for regions
 Run `python manage.py update-region-bbox` to update the bbox for each region in the database.
 
@@ -227,5 +245,13 @@ Run ` python manage.py update-sovereign-and-disputed new_fields.csv` to update t
 ## Update Mapbox Tilesets
 To update GO countries and districts Mapbox tilesets, run the management command `python manage.py update-mapbox-tilesets`. This will export all country and district geometries to a GeoJSON file, and then upload them to Mapbox. The tilesets will take a while to process. The updated status can be viewed on the Mapbox Studio under tilesets. To run this management command, MAPBOX_ACCESS_TOKEN should be set in the environment.
 
+### Options available for the command
+* `--production` — update production tilesets. If this flag is not set, by default the script will only update staging tiles
+* `--update-countries` — update tileset for countries, including labels
+* `--update-districts` — update tileset for districts, including labels
+* `--update-all` — update all countries and districts tilesets
+* `--create-and-update-admin2 <ISO3>` — if a new admin2 tileset should be created, use this argument. It will create a new source on Mapbox and then register a tileset. Ensure that a recipe is create in `mapbox/admin2/` directory. For example, see `mapbox/admin2/COL.json`. To run `python manage.py update-mapbox-tilesets --create-and-update-admin2 COL`
+* `--update-admin2 <ISO3>` — use this to update an existing admin2 tileset. For example, `python manage.py update-mapbox-tilesets --update-admin2 COL`
+
 ## Import GEC codes
 To import GEC codes along with country ids, run `python manage.py import-gec-code appeal_ingest_match.csv`. The CSV should have the columns `'GST_code', 'GST_name', 'GO ID', 'ISO'`

api/admin.py

@@ -550,6 +550,9 @@ class RegionAdmin(geoadmin.OSMGeoAdmin, CompareVersionAdmin, RegionRestrictedAdm
     search_fields = ('name',)
     modifiable = True
 
+class Admin2Admin(geoadmin.OSMGeoAdmin, CompareVersionAdmin, RegionRestrictedAdmin):
+    search_fields = ('name',)
+    modifiable = True
 
 class UserProfileAdmin(CompareVersionAdmin):
     search_fields = ('user__username', 'user__email', 'country__name',)
@@ -795,6 +798,7 @@ def has_add_permission(cls, request, obj=None):
 admin.site.register(models.Country, CountryAdmin)
 admin.site.register(models.Region, RegionAdmin)
 admin.site.register(models.District, DistrictAdmin)
+admin.site.register(models.Admin2, Admin2Admin)
 admin.site.register(models.Appeal, AppealAdmin)
 admin.site.register(models.AppealDocument, AppealDocumentAdmin)
 admin.site.register(models.AppealFilter, AppealFilterAdmin)

api/management/commands/import-admin2-data.py

@@ -13,7 +13,7 @@
 from api.models import Admin2Geoms
 
 class Command(BaseCommand):
-  help = "import a shapefile of administrative boundary level 2 data to the GO database. To run, python manage.py import-admin2-data input.shp --country-iso2=af"
+  help = "import a shapefile of administrative boundary level 2 data to the GO database. To run, python manage.py import-admin2-data input.shp"
 
   missing_args_message = "Filename is missing. A shapefile with valid admin polygons is required."
 
@@ -43,12 +43,6 @@ def add_arguments(self, parser):
       action='store_true',
       help='Import all  admin2 boundaries in the shapefile, if possible.'
     )
-    parser.add_argument(
-      '--country-iso2',
-      type=str,
-      required=True,
-      help='Country iso2 code'
-    )
 
   @transaction.atomic
   def handle(self, *args, **options):
@@ -77,13 +71,18 @@ def handle(self, *args, **options):
 
       # loop through each feature in the shapefile
       for feature in data[0]:
-          code = feature.get("code")
-          name = feature.get("name")
+          code = feature.get("code") if 'code' in feature.fields else feature.get("pcode")
+          name = feature.get("name") if 'name' in feature.fields else feature.get("shapeName")
+          admin1_id = feature.get("district_id") if "district_id" in feature.fields else feature.get("admin1_id")
+          local_name = feature.get("local_name") if "local_name" in feature.fields else None
+          local_name_code = feature.get("local_name_code") if "local_name_code" in feature.fields else None
+          alternate_name = feature.get("alternate_name") if "alternate_name" in feature.fields else None
+          alternate_name_code = feature.get("alternate_name_code") if "alternate_name_code" in feature.fields else None
+
+          #FIXME: must make sure code and admin1_id are not null before continuing
+
           geom_wkt = feature.geom.wkt
           geom = GEOSGeometry(geom_wkt, srid=4326)
-          if geom.geom_type == "Polygon":
-              geom = MultiPolygon(geom)
-
           centroid = geom.centroid.wkt
           bbox = geom.envelope.wkt
           # import all shapes for admin2
@@ -117,26 +116,35 @@ def handle(self, *args, **options):
 
   @transaction.atomic
   def add_admin2(self, options, import_missing, feature, geom, centroid, bbox):
-      code = feature.get("code") or "N.A"
-      name = feature.get("name")
+      code = feature.get("code") if 'code' in feature.fields else feature.get("pcode")
+      name = feature.get("name") if 'name' in feature.fields else feature.get("shapeName")
+      admin1_id = feature.get("district_id") if "district_id" in feature.fields else feature.get("admin1_id")
+      local_name = feature.get("local_name") if "local_name" in feature.fields else None
+      local_name_code = feature.get("local_name_code") if "local_name_code" in feature.fields else None
+      alternate_name = feature.get("alternate_name") if "alternate_name" in feature.fields else None
+      alternate_name_code = feature.get("alternate_name_code") if "alternate_name_code" in feature.fields else None
       admin2 = Admin2()
       admin2.code = code
       admin2.name = name
       admin2.centroid = centroid
       admin2.bbox = bbox
-      country_iso2 = options["country_iso2"]
-      # find district_id based on centroid of admin2 and country.
+      admin2.local_name = local_name
+      admin2.local_name_code = local_name_code
+      admin2.alternate_name = alternate_name
+      admin2.alternate_name_code = alternate_name_code
+
       try:
-          admin2.admin1_id = self.find_district_id(centroid, country_iso2)
+        admin1 = District.objects.get(id=admin1_id)
+        admin2.admin1_id = admin1.id
       except ObjectDoesNotExist:
-          print(f"Country({country_iso2}) or admin 1 does not found for - admin2: {name}")
+          print(f"admin1 {admin1_id} not found for - admin2: {name}")
           pass
 
       # save data
       if admin2.admin1_id is not None and ((import_missing == "all") or (code in import_missing.keys())):
           try:
-              admin2.save()
               print("importing", admin2.name)
+              admin2.save()
               if options["update_geom"]:
                   self.update_geom(admin2, geom)
           except IntegrityError as e: