Skip to content

Commit 93b2307

Browse files
geohackergeohacker
authored
Merge pull request #1575 from IFRCGo/admin2-mvt
admin2 vector tiles + better tile management commands
2 parents a686800 + 33c7fd4 commit 93b2307

File tree

8 files changed

+291
-42
lines changed

8 files changed

+291
-42
lines changed

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,5 +247,13 @@ Run ` python manage.py update-sovereign-and-disputed new_fields.csv` to update t
247247
## Update Mapbox Tilesets
248248
To update GO countries and districts Mapbox tilesets, run the management command `python manage.py update-mapbox-tilesets`. This will export all country and district geometries to a GeoJSON file, and then upload them to Mapbox. The tilesets will take a while to process. The updated status can be viewed on the Mapbox Studio under tilesets. To run this management command, MAPBOX_ACCESS_TOKEN should be set in the environment.
249249

250+
### Options available for the command
251+
* `--production` — update production tilesets. If this flag is not set, by default the script will only update staging tiles
252+
* `--update-countries` — update tileset for countries, including labels
253+
* `--update-districts` — update tileset for districts, including labels
254+
* `--update-all` — update all countries and districts tilesets
255+
* `--create-and-update-admin2 <ISO3>` — if a new admin2 tileset should be created, use this argument. It will create a new source on Mapbox and then register a tileset. Ensure that a recipe is create in `mapbox/admin2/` directory. For example, see `mapbox/admin2/COL.json`. To run `python manage.py update-mapbox-tilesets --create-and-update-admin2 COL`
256+
* `--update-admin2 <ISO3>` — use this to update an existing admin2 tileset. For example, `python manage.py update-mapbox-tilesets --update-admin2 COL`
257+
250258
## Import GEC codes
251259
To import GEC codes along with country ids, run `python manage.py import-gec-code appeal_ingest_match.csv`. The CSV should have the columns `'GST_code', 'GST_name', 'GO ID', 'ISO'`

‎api/management/commands/update-mapbox-tilesets.py‎

Lines changed: 221 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -2,75 +2,254 @@
22
from django.core.management.base import BaseCommand, CommandError
33
from django.conf import settings
44
import time
5+
import os
56

67
class Command(BaseCommand):
78
help = "This command produces a countries.geojson and districts.geojson, and uploads them to Mapbox. It is the source for all GO Maps."
89

10+
missing_args_message = "Argument missing. Specify --update-countries, --update-districts or --update-all."
11+
12+
def add_arguments(self, parser):
13+
parser.add_argument(
14+
'--update-countries',
15+
action='store_true',
16+
help='Update tileset for countries'
17+
)
18+
parser.add_argument(
19+
'--update-districts',
20+
action='store_true',
21+
help='Update tileset for districts'
22+
)
23+
parser.add_argument(
24+
'--create-and-update-admin2',
25+
help='Create and update admin2 tileset for this country ISO'
26+
)
27+
parser.add_argument(
28+
'--update-admin2',
29+
help='Update admin2 tileset for this country ISO'
30+
)
31+
parser.add_argument(
32+
'--update-all',
33+
action='store_true',
34+
help='Update tileset for countries and districts'
35+
)
36+
parser.add_argument(
37+
'--production',
38+
action='store_true',
39+
help='Update production tilesets. Default is staging'
40+
)
41+
42+
db = settings.DATABASES['default']
43+
DB_HOST = db['HOST']
44+
DB_NAME = db['NAME']
45+
DB_USER = db['USER']
46+
DB_PASSWORD = db['PASSWORD']
47+
DB_PORT = 5432
48+
connection_string = 'PG:host={} dbname={} user={} password={} port={}'.format(DB_HOST, DB_NAME, DB_USER, DB_PASSWORD, DB_PORT)
49+
950
def handle(self, *args, **options):
1051
try:
11-
db = settings.DATABASES['default']
12-
DB_HOST = db['HOST']
13-
DB_NAME = db['NAME']
14-
DB_USER = db['USER']
15-
DB_PASSWORD = db['PASSWORD']
16-
DB_PORT = 5432
17-
connection_string = 'PG:host={} dbname={} user={} password={} port={}'.format(DB_HOST, DB_NAME, DB_USER, DB_PASSWORD, DB_PORT)
1852

53+
if (os.getenv("MAPBOX_ACCESS_TOKEN") is None):
54+
raise Exception('MAPBOX_ACCESS_TOKEN must be set')
55+
56+
staging = True
57+
if (options['production']):
58+
staging = False
59+
60+
if options['update_countries'] or options['update_all']:
61+
self.update_countries(staging)
62+
63+
if options['update_districts'] or options['update_all']:
64+
self.update_districts(staging)
65+
66+
if options['create_and_update_admin2']:
67+
print(f'Creating source for {options["create_and_update_admin2"]}')
68+
self.create_and_update_admin2(options['create_and_update_admin2'], staging)
69+
70+
if options['update_admin2']:
71+
print(f'Updating tileset for {options["update_admin2"]}')
72+
self.update_admin2(options['update_admin2'], staging)
73+
74+
except BaseException as e:
75+
raise CommandError('Could not update tilesets: ' + str(e))
76+
77+
def update_countries(self, staging):
78+
try:
1979
print('Exporting countries...')
20-
subprocess.check_call(['touch', '/tmp/countries.geojson'])
21-
subprocess.check_call(['rm', '/tmp/countries.geojson'])
22-
subprocess.check_call(['ogr2ogr', '-f', 'GeoJSON', '/tmp/countries.geojson', connection_string, '-sql', 'select cd.country_id, cd.geom, c.name, c.name_es, c.name_fr, c.name_ar, c.iso, c.region_id, c.iso3, c.independent, c.is_deprecated, c.disputed, c.fdrs, c.record_type from api_countrygeoms cd, api_country c where cd.country_id = c.id and c.record_type=1' ])
23-
print('Countries written to /tmp/countries.geojson')
2480

25-
print('Exporting districts...')
26-
subprocess.check_call(['touch', '/tmp/districts.geojson'])
27-
subprocess.check_call(['rm', '/tmp/districts.geojson'])
28-
# FIXME eventually should be name_en, name_es etc.
29-
subprocess.check_call(['ogr2ogr', '-lco', 'COORDINATE_PRECISION=5', '-f', 'GeoJSON', '/tmp/districts.geojson', connection_string, '-sql', 'select cd.district_id, cd.geom, c.name, c.code, c.country_id, c.is_enclave, c.is_deprecated, country.iso as country_iso, country.iso3 as country_iso3, country.name as country_name, country.name_es as country_name_es, country.name_fr as country_name_fr, country.name_ar as country_name_ar from api_districtgeoms cd, api_district c, api_country country where cd.district_id = c.id and cd.geom is not null and country.id=c.country_id' ])
30-
print('Districts written to /tmp/districts.geojson')
81+
try:
82+
os.remove(f'/tmp/countries.geojson')
83+
except FileNotFoundError as e:
84+
pass
3185

86+
subprocess.check_call(['ogr2ogr', '-f', 'GeoJSON', '/tmp/countries.geojson', self.connection_string, '-sql', 'select cd.country_id, cd.geom, c.name, c.name_es, c.name_fr, c.name_ar, c.iso, c.region_id, c.iso3, c.independent, c.is_deprecated, c.disputed, c.fdrs, c.record_type from api_countrygeoms cd, api_country c where cd.country_id = c.id and c.record_type=1' ])
87+
print('Countries written to /tmp/countries.geojson')
88+
except Exception as e:
89+
print('Failed to export countries', e)
90+
raise
91+
92+
try:
3293
print('Exporting country centroids...')
33-
subprocess.check_call(['touch', '/tmp/country-centroids.geojson'])
34-
subprocess.check_call(['rm', '/tmp/country-centroids.geojson'])
35-
subprocess.check_call(['ogr2ogr', '-lco', 'COORDINATE_PRECISION=4', '-f', 'GeoJSON', '/tmp/country-centroids.geojson', connection_string, '-sql', 'select id as country_id, name_en as name, name_ar, name_es, name_fr, independent, disputed, is_deprecated, iso, iso3, record_type, fdrs, region_id, centroid from api_country where centroid is not null'])
3694

37-
print('Exporting district centroids...')
38-
subprocess.check_call(['touch', '/tmp/district-centroids.geojson'])
39-
subprocess.check_call(['rm', '/tmp/district-centroids.geojson'])
40-
# FIXME eventually should be name_en, name_es etc.
41-
subprocess.check_call(['ogr2ogr', '-lco', 'COORDINATE_PRECISION=4', '-f', 'GeoJSON', '/tmp/district-centroids.geojson', connection_string, '-sql', 'select d.id as district_id, d.country_id as country_id, d.name, d.code, d.is_deprecated, d.is_enclave, c.iso as country_iso, c.iso3 as country_iso3, c.name as country_name, c.name_es as country_name_es, c.name_fr as country_name_fr, c.name_ar as country_name_ar, d.centroid from api_district d join api_country c on d.country_id=c.id where d.centroid is not null'])
95+
try:
96+
os.remove(f'/tmp/country-centroids.geojson')
97+
except FileNotFoundError as e:
98+
pass
4299

100+
subprocess.check_call(['ogr2ogr', '-lco', 'COORDINATE_PRECISION=4', '-f', 'GeoJSON', '/tmp/country-centroids.geojson', self.connection_string, '-sql', 'select id as country_id, name_en as name, name_ar, name_es, name_fr, independent, disputed, is_deprecated, iso, iso3, record_type, fdrs, region_id, centroid from api_country where centroid is not null'])
101+
except Exception as e:
102+
print('Failed to export country centroids', e)
103+
raise
43104

105+
try:
44106
print('Update Mapbox tileset source for countries...')
45-
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', 'go-countries-src', '/tmp/countries.geojson'])
107+
tileset_source_name = f'go-countries-src-staging' if staging else f'go-countries-src'
108+
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', tileset_source_name, '/tmp/countries.geojson'])
109+
except Exception as e:
110+
print('Failed to update tileset source for countries', e)
111+
raise
46112

113+
try:
47114
print('Update Mapbox tileset for countries... and sleeping a minute')
48-
subprocess.check_call(['tilesets', 'publish', 'go-ifrc.go-countries'])
115+
tileset_name = f'go-ifrc.go-countries-staging' if staging else f'go-ifrc.go-countries'
116+
subprocess.check_call(['tilesets', 'publish', tileset_name])
49117
time.sleep(60)
118+
except Exception as e:
119+
print('Failed to update tileset for countries', e)
120+
raise
50121

51-
52-
print('Update Mapbox tileset source for districts...')
53-
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', 'go-districts-src-1', '/tmp/districts.geojson'])
54-
55-
print('Update Mapbox tileset for districts... and sleeping a minute')
56-
subprocess.check_call(['tilesets', 'publish', 'go-ifrc.go-districts-1'])
57-
time.sleep(60)
58-
59-
122+
try:
60123
print('Update Mapbox tileset source for country centroids...')
124+
tileset_source_name = f'go-country-centroids-staging' if staging else f'go-country-centroids'
61125
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', 'go-country-centroids', '/tmp/country-centroids.geojson'])
126+
except Exception as e:
127+
print('Failed to update tileset source for country centroids')
128+
raise
62129

130+
try:
63131
print('Update Mapbox tileset for country centroids... and sleeping a minute')
64-
subprocess.check_call(['tilesets', 'publish', 'go-ifrc.go-country-centroids'])
132+
tileset_name = f'go-ifrc.go-country-centroids-staging' if staging else f'go-ifrc.go-country-centroids'
133+
subprocess.check_call(['tilesets', 'publish', tileset_name])
65134
time.sleep(60)
135+
except Exception as e:
136+
print('Failed to update tileset for country centroids')
137+
raise
66138

67139

68-
print('Update Mapbox tileset source for district centroids...')
69-
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', 'go-district-centroids', '/tmp/district-centroids.geojson'])
140+
def update_districts(self, staging):
141+
try:
142+
print('Exporting districts...')
143+
144+
try:
145+
os.remove(f'/tmp/distrcits.geojson')
146+
except FileNotFoundError as e:
147+
pass
148+
149+
# FIXME eventually should be name_en, name_es etc.
150+
subprocess.check_call(['ogr2ogr', '-lco', 'COORDINATE_PRECISION=5', '-f', 'GeoJSON', '/tmp/districts.geojson', self.connection_string, '-sql', 'select cd.district_id, cd.geom, c.name, c.code, c.country_id, c.is_enclave, c.is_deprecated, country.iso as country_iso, country.iso3 as country_iso3, country.name as country_name, country.name_es as country_name_es, country.name_fr as country_name_fr, country.name_ar as country_name_ar from api_districtgeoms cd, api_district c, api_country country where cd.district_id = c.id and cd.geom is not null and country.id=c.country_id' ])
151+
print('Districts written to /tmp/districts.geojson')
152+
except Exception as e:
153+
print('Failed to export districts', e)
154+
raise
155+
156+
try:
157+
print('Exporting district centroids...')
158+
159+
try:
160+
os.remove(f'/tmp/district-centroids.geojson')
161+
except FileNotFoundError as e:
162+
pass
163+
164+
# FIXME eventually should be name_en, name_es etc.
165+
subprocess.check_call(['ogr2ogr', '-lco', 'COORDINATE_PRECISION=4', '-f', 'GeoJSON', '/tmp/district-centroids.geojson', self.connection_string, '-sql', 'select d.id as district_id, d.country_id as country_id, d.name, d.code, d.is_deprecated, d.is_enclave, c.iso as country_iso, c.iso3 as country_iso3, c.name as country_name, c.name_es as country_name_es, c.name_fr as country_name_fr, c.name_ar as country_name_ar, d.centroid from api_district d join api_country c on d.country_id=c.id where d.centroid is not null'])
166+
except Exception as e:
167+
print('Failed to export district centroids', e)
168+
raise
169+
170+
try:
171+
print('Update Mapbox tileset source for districts...')
172+
tileset_source_name = f'go-districts-src-staging' if staging else f'go-districts-src-1'
173+
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', tileset_source_name, '/tmp/districts.geojson'])
174+
except Exception as e:
175+
print('Failed to update tileset source for districts', e)
176+
raise
177+
178+
try:
179+
print('Update Mapbox tileset for districts... and sleeping a minute')
180+
tileset_name = f'go-ifrc.go-districts-staging' if staging else f'go-ifrc.go-districts-1'
181+
subprocess.check_call(['tilesets', 'publish', tileset_name])
182+
time.sleep(60)
183+
except Exception as e:
184+
print('Failed to update tileset for districts')
185+
raise
70186

187+
try:
188+
print('Update Mapbox tileset source for district centroids...')
189+
tileset_source_name = f'go-district-centroids-staging' if staging else f'go-district-centroids'
190+
subprocess.check_call(['tilesets', 'upload-source', '--replace', 'go-ifrc', tileset_source_name, '/tmp/district-centroids.geojson'])
191+
except Exception as e:
192+
print('Failed to update tileset source for district centroid')
193+
raise
194+
try:
71195
print('Update Mapbox tileset for district centroids... [no sleep]')
72-
subprocess.check_call(['tilesets', 'publish', 'go-ifrc.go-district-centroids'])
196+
tileset_name = f'go-ifrc.go-district-centroids-staging' if staging else f'go-ifrc.go-district-centroids'
197+
subprocess.check_call(['tilesets', 'publish', tileset_name])
198+
except Exception as e:
199+
print('Failed to update tileset for distrct centroids')
200+
raise
201+
202+
def create_and_update_admin2(self, iso, staging=True):
203+
# create a new tileset source
204+
205+
status = self.prepare_admin2_geojson(iso)
206+
if (status):
207+
update_status = self.update_admin2(iso, staging)
208+
if (update_status):
209+
tileset_name = f'go-ifrc.go-admin2-{iso}-staging'
210+
recipe_name = f'mapbox/admin2/{iso}-staging.json'
211+
if not staging:
212+
tileset_name = f'go-ifrc.go-admin2-{iso}'
213+
recipe_name = f'mapbox/admin2/{iso}.json'
214+
215+
create_status = subprocess.run(['tilesets', 'create', tileset_name, '--recipe', recipe_name, '--name', f'GO Admin2 {iso}'])
216+
if create_status:
217+
publish_status = self.publish_admin2(iso, staging, create=True)
218+
return publish_status
219+
220+
def update_admin2(self, iso, staging=True):
221+
# update tileset source
222+
# update tileset and publish
223+
tileset_source__name = f'go-admin2-{iso}-src-staging'
224+
if not staging:
225+
tileset_source__name = f'go-admin2-{iso}-src'
226+
227+
status = subprocess.run(['tilesets', 'upload-source', '--replace', 'go-ifrc', tileset_source__name, f'/tmp/{iso}.geojson'])
228+
return True if status.returncode == 0 else False
229+
230+
def publish_admin2(self, iso, staging=True, create=False):
231+
if (not create):
232+
update_status = self.update_admin2(iso, staging)
233+
else:
234+
update_status = True
235+
236+
if update_status:
237+
tileset_name = f'go-ifrc.go-admin2-{iso}-staging'
238+
if not staging:
239+
tileset_name = f'go-ifrc.go-admin2-{iso}'
240+
241+
publish_status = subprocess.run(['tilesets', 'publish', tileset_name])
242+
return True if publish_status.returncode == 0 else False
243+
else:
244+
return False
245+
246+
def prepare_admin2_geojson(self, iso):
247+
# query the database and create geojson
248+
try:
249+
os.remove(f'/tmp/{iso}.geojson')
250+
except FileNotFoundError as e:
251+
pass
73252

253+
status = subprocess.run(['ogr2ogr', '-f', 'GeoJSON', f'/tmp/{iso}.geojson', self.connection_string, '-sql', f'select d.id as admin1_id, d.name as admin1_name, ad.name, ad.id, adg.geom from api_country as c, api_district as d, api_admin2 as ad, api_admin2geoms as adg where c.id=d.country_id and c.iso3=\'{iso}\' and ad.admin1_id=d.id and adg.admin2_id = ad.id'])
74254

75-
except BaseException as e:
76-
raise CommandError('Could not update tilesets: ' + str(e))
255+
return True if status.returncode == 0 else False

‎mapbox/admin2/COL-staging.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"version": 1,
3+
"layers": {
4+
"go-admin2-COL-staging": {
5+
"source": "mapbox://tileset-source/go-ifrc/go-admin2-COL-src-staging",
6+
"minzoom": 10,
7+
"maxzoom": 14
8+
}
9+
}
10+
}
11+

‎mapbox/admin2/COL.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"version": 1,
3+
"layers": {
4+
"go-admin2-COL": {
5+
"source": "mapbox://tileset-source/go-ifrc/go-admin2-COL-src",
6+
"minzoom": 10,
7+
"maxzoom": 14
8+
}
9+
}
10+
}
11+

‎mapbox/countries-recipe-staging.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"version": 1,
3+
"layers": {
4+
"go-countries-staging": {
5+
"source": "mapbox://tileset-source/go-ifrc/go-countries-src-staging",
6+
"minzoom": 0,
7+
"maxzoom": 10
8+
}
9+
}
10+
}

‎mapbox/country-centroids-recipe-staging.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"version": 1,
3+
"layers": {
4+
"go-country-centroids-staging": {
5+
"source": "mapbox://tileset-source/go-ifrc/go-country-centroids-staging",
6+
"minzoom": 0,
7+
"maxzoom": 10
8+
}
9+
}
10+
}

‎mapbox/district-centroids-recipe-staging.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"version": 1,
3+
"layers": {
4+
"go-district-centroids-staging": {
5+
"source": "mapbox://tileset-source/go-ifrc/go-district-centroids-staging",
6+
"minzoom": 3,
7+
"maxzoom": 10
8+
}
9+
}
10+
}

‎mapbox/districts-recipe-staging.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"version": 1,
3+
"layers": {
4+
"go-districts-staging": {
5+
"source": "mapbox://tileset-source/go-ifrc/go-districts-src-staging",
6+
"minzoom": 3,
7+
"maxzoom": 10
8+
}
9+
}
10+
}

0 commit comments

Comments
 (0)