Skip to content

Commit a47ae44

Browse files
c-wc-w
committed
Add importer for Divipola data
After running the importer, we have just under 6k new features in the database. Most of these features come from the centro-poplado datasets and therewith represent small towns or cities. ``` features=# select count(*) from features; count ------- 5879 features=# select count(*), layer from features group by layer order by count desc; count | layer -------+---------- 4754 | locality 1093 | county 32 | region ``` Unfortunately the Divipola data doesn't give us much additional information on top of just the name, type and geometry of the features, so additional fields like population will not be populated in the featureService. Resolves #14
1 parent e666748 commit a47ae44

File tree

1 file changed

+191
-0
lines changed

1 file changed

+191
-0
lines changed

‎tools/importDivipola.js‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
To generate the data files that get imported into the featureService by this script,
3+
please execute the following helper script:
4+
5+
```
6+
#!/usr/bin/env bash
7+
set -e
8+
9+
mkdir divipola && cd divipola
10+
11+
geojson_dir='geojson'
12+
shapefile_dir='shp'
13+
14+
# The list of URLs below was generated via the following steps:
15+
# 1) Go to https://geoportal.dane.gov.co/v2/?page=elementoDescargaMGN and switch to the 'Vigencia 2012' tab
16+
# 2) Open the Javascript console and execute:
17+
#
18+
# var zips = new Set();
19+
# var links = document.getElementsByTagName('a');
20+
# for (var i = 0; i < links.length; i++) {
21+
# var link = links[i];
22+
# if (link.href.indexOf('.zip') !== -1 &&
23+
# link.href.indexOf('MGN2012_') !== -1) {
24+
# zips.add(link.href);
25+
# }
26+
# }
27+
# console.log(JSON.stringify(Array.from(zips)))
28+
#
29+
# 3) Copy-paste console output into the variable
30+
#
31+
shapefile_urls='http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_91_AMAZONAS.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_05_ANTIOQUIA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_81_ARAUCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_88_SAN_ANDRES.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_08_ATLANTICO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_11_BOGOTA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_13_BOLIVAR.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_15_BOYACA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_17_CALDAS.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_18_CAQUETA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_85_CASANARE.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_19_CAUCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_20_CESAR.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_27_CHOCO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_23_CORDOBA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_25_CUNDINAMARCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_94_GUAINIA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_95_GUAVIARE.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_41_HUILA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_44_LA_GUAJIRA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_47_MAGDALENA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_50_META.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_52_NARI%c3%91O.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_54_NORTE_SANTANDER.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_86_PUTUMAYO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_63_QUINDIO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_66_RISARALDA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_68_SANTANDER.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_70_SUCRE.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_73_TOLIMA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_76_VALLE_CAUCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_97_VAUPES.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_99_VICHADA.zip'
32+
33+
echo 'Now downloading shapefiles...'
34+
if ! (command -v curl >/dev/null); then sudo apt-get install -y curl > /dev/null; fi
35+
if ! (command -v unzip >/dev/null); then sudo apt-get install -y unzip > /dev/null; fi
36+
mkdir -p '$shapefile_dir' > /dev/null
37+
cd '$shapefile_dir'
38+
echo '$shapefile_urls' | tr ',' '\n' | while read shapefile_url; do
39+
curl -sLO '$shapefile_url'
40+
done
41+
for shapefile in *.zip; do
42+
unzip '$shapefile'
43+
done
44+
cd -
45+
46+
echo 'Now converting to geojson...'
47+
if ! (command -v ogr2ogr >/dev/null); then (sudo add-apt-repository -y ppa:ubuntugis/ppa && sudo apt-get update && sudo apt-get install -y gdal-bin) > /dev/null; fi
48+
mkdir -p '$geojson_dir' > /dev/null
49+
find '$shapefile_dir' -type d -name '[0-9]*' | while read region_path; do
50+
region_name='$(basename $region_path)'
51+
find '$region_path/ADMINISTRATIVO' -type f -name '*.shp' | while read shp_path; do
52+
shp_type='$(basename $shp_path .shp)'
53+
ogr2ogr -f 'GeoJSON' -t_srs 'crs:84' '$geojson_dir/$region_name-$shp_type.geojson' '$shp_path'
54+
done
55+
done
56+
```
57+
58+
Note that this is a quite slow process and may take several hours to complete since the script downloads
59+
and munges over 100 GB of data.
60+
*/
61+
62+
'use strict';
63+
64+
const fs = require('fs');
65+
const path = require('path');
66+
const services = require('../services');
67+
68+
const DATA_LANGUAGE = 'es';
69+
70+
function processFeature(placeType, geojson, i) {
71+
const type = geojson.type;
72+
if (type !== 'Feature') {
73+
return Promise.reject(`feature ${i} has bad feature type: ${type}`);
74+
}
75+
76+
const geometry = geojson.geometry;
77+
if (!geometry) {
78+
return Promise.reject(`feature ${i} has no geometry`);
79+
}
80+
81+
let id;
82+
if (placeType === 'locality') {
83+
id = geojson.properties.CPOB_CCNCT;
84+
} else if (placeType === 'county') {
85+
id = `${geojson.properties.DPTO_CCDGO}${geojson.properties.MPIO_CCDGO}`;
86+
} else if (placeType === 'region') {
87+
id = geojson.properties.DPTO_CCDGO;
88+
}
89+
if (!id) {
90+
return Promise.reject(`feature ${i} has no id`);
91+
}
92+
93+
const name = geojson.properties.CPOB_CNMBR || geojson.properties.MPIO_CNMBR || geojson.properties.DPTO_CNMBR;
94+
if (!name) {
95+
return Promise.reject(`feature ${i} has no name`);
96+
}
97+
98+
const feature = {
99+
id: `divipola-${id}`,
100+
name: name,
101+
layer: placeType,
102+
hull: geometry,
103+
properties: {
104+
names: {
105+
DATA_LANGUAGE: name,
106+
},
107+
tags: [
108+
'boundary:administrative',
109+
`placetype:${placeType}`
110+
]
111+
}
112+
};
113+
114+
return new Promise((resolve, reject) => {
115+
services.features.upsert(feature, (err) => {
116+
if (err) return reject(err);
117+
118+
console.log(`done with feature ${i}: ${feature.id},${feature.name}`);
119+
resolve();
120+
});
121+
});
122+
}
123+
124+
function processFile(file) {
125+
return new Promise((resolve, reject) => {
126+
fs.readFile(file, (err, data) => {
127+
if (err) return reject(`error reading file: ${err}`);
128+
129+
let featureCollection;
130+
try {
131+
featureCollection = JSON.parse(data);
132+
} catch(e) {
133+
return reject(`failed to parse: ${file}`);
134+
}
135+
136+
const type = featureCollection.type;
137+
if (type !== 'FeatureCollection') {
138+
return reject(`not a feature collection: ${file}`);
139+
}
140+
141+
const features = featureCollection.features;
142+
if (!features || !features.length) {
143+
return reject(`no features in collection: ${file}`);
144+
}
145+
146+
let placeType;
147+
if (file.indexOf('MGN_RUR_CENTRO_POBLADO') !== -1) {
148+
placeType = 'locality';
149+
} else if (file.indexOf('MGN_ADM_MPIO_GRAFICO') !== -1) {
150+
placeType = 'county';
151+
} else if (file.indexOf('MGN_ADM_DPTO_POLITICO') !== -1) {
152+
placeType = 'region';
153+
} else {
154+
return reject(`unable to determine placeType for ${file}`);
155+
}
156+
157+
Promise.all(features.map((feature, i) => processFeature(placeType, feature, i)))
158+
.then(resolve)
159+
.catch(console.log);
160+
});
161+
});
162+
}
163+
164+
function processDirectory(dir) {
165+
return new Promise((resolve, reject) => {
166+
fs.readdir(dir, (err, files) => {
167+
if (err) return reject(err);
168+
169+
Promise.all(files.map(file => processFile(path.resolve(dir, file))))
170+
.then(resolve)
171+
.catch(console.log);
172+
});
173+
});
174+
};
175+
176+
services.init(err => {
177+
if (err) {
178+
console.log(err);
179+
process.exit(1);
180+
}
181+
182+
processDirectory('./divipola/geojson')
183+
.then(() => {
184+
console.log('all done');
185+
process.exit(0);
186+
})
187+
.catch((err) => {
188+
console.log(`finished with error: ${err}`);
189+
process.exit(1);
190+
});
191+
});

0 commit comments

Comments
 (0)