Skip to content

Commit 503d4b5

Browse files
c-wc-w
authored
Merge pull request #16 from CatalystCode/load-divipola
Add importer for Divipola datasets
2 parents d9b4909 + a47ae44 commit 503d4b5

File tree

3 files changed

+193
-2
lines changed

3 files changed

+193
-2
lines changed

‎controllers/features.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ exports.getByPoint = function(req, res) {
5353

5454
exports.getByName = function(req, res) {
5555
let query = {
56-
name: req.params.name.split(','),
56+
name: req.params.name.split(/,(?=[^ ])/),
5757
include: req.query.include
5858
};
5959

‎services/features.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ function getByName(query, callback) {
159159
const names = query.name.constructor === Array ? query.name : [query.name];
160160

161161
let namesDisjunction = `(${names.map(function(name) {
162-
return `lower(name) = ${escapeSql(name.toLowerCase())}`;
162+
return `lower(name) = lower(${escapeSql(name)})`;
163163
}).join(" OR ")})`;
164164
let nameQuery = `SELECT ${buildQueryColumns(query)} FROM features WHERE ${namesDisjunction}`;
165165

‎tools/importDivipola.js‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
To generate the data files that get imported into the featureService by this script,
3+
please execute the following helper script:
4+
5+
```
6+
#!/usr/bin/env bash
7+
set -e
8+
9+
mkdir divipola && cd divipola
10+
11+
geojson_dir='geojson'
12+
shapefile_dir='shp'
13+
14+
# The list of URLs below was generated via the following steps:
15+
# 1) Go to https://geoportal.dane.gov.co/v2/?page=elementoDescargaMGN and switch to the 'Vigencia 2012' tab
16+
# 2) Open the Javascript console and execute:
17+
#
18+
# var zips = new Set();
19+
# var links = document.getElementsByTagName('a');
20+
# for (var i = 0; i < links.length; i++) {
21+
# var link = links[i];
22+
# if (link.href.indexOf('.zip') !== -1 &&
23+
# link.href.indexOf('MGN2012_') !== -1) {
24+
# zips.add(link.href);
25+
# }
26+
# }
27+
# console.log(JSON.stringify(Array.from(zips)))
28+
#
29+
# 3) Copy-paste console output into the variable
30+
#
31+
shapefile_urls='http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_91_AMAZONAS.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_05_ANTIOQUIA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_81_ARAUCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_88_SAN_ANDRES.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_08_ATLANTICO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_11_BOGOTA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_13_BOLIVAR.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_15_BOYACA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_17_CALDAS.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_18_CAQUETA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_85_CASANARE.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_19_CAUCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_20_CESAR.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_27_CHOCO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_23_CORDOBA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_25_CUNDINAMARCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_94_GUAINIA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_95_GUAVIARE.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_41_HUILA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_44_LA_GUAJIRA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_47_MAGDALENA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_50_META.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_52_NARI%c3%91O.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_54_NORTE_SANTANDER.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_86_PUTUMAYO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_63_QUINDIO.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_66_RISARALDA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_68_SANTANDER.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_70_SUCRE.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_73_TOLIMA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_76_VALLE_CAUCA.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_97_VAUPES.zip','http://www.dane.gov.co/DescargaMGN/Departamento/MGN2012_99_VICHADA.zip'
32+
33+
echo 'Now downloading shapefiles...'
34+
if ! (command -v curl >/dev/null); then sudo apt-get install -y curl > /dev/null; fi
35+
if ! (command -v unzip >/dev/null); then sudo apt-get install -y unzip > /dev/null; fi
36+
mkdir -p '$shapefile_dir' > /dev/null
37+
cd '$shapefile_dir'
38+
echo '$shapefile_urls' | tr ',' '\n' | while read shapefile_url; do
39+
curl -sLO '$shapefile_url'
40+
done
41+
for shapefile in *.zip; do
42+
unzip '$shapefile'
43+
done
44+
cd -
45+
46+
echo 'Now converting to geojson...'
47+
if ! (command -v ogr2ogr >/dev/null); then (sudo add-apt-repository -y ppa:ubuntugis/ppa && sudo apt-get update && sudo apt-get install -y gdal-bin) > /dev/null; fi
48+
mkdir -p '$geojson_dir' > /dev/null
49+
find '$shapefile_dir' -type d -name '[0-9]*' | while read region_path; do
50+
region_name='$(basename $region_path)'
51+
find '$region_path/ADMINISTRATIVO' -type f -name '*.shp' | while read shp_path; do
52+
shp_type='$(basename $shp_path .shp)'
53+
ogr2ogr -f 'GeoJSON' -t_srs 'crs:84' '$geojson_dir/$region_name-$shp_type.geojson' '$shp_path'
54+
done
55+
done
56+
```
57+
58+
Note that this is a quite slow process and may take several hours to complete since the script downloads
59+
and munges over 100 GB of data.
60+
*/
61+
62+
'use strict';
63+
64+
const fs = require('fs');
65+
const path = require('path');
66+
const services = require('../services');
67+
68+
const DATA_LANGUAGE = 'es';
69+
70+
function processFeature(placeType, geojson, i) {
71+
const type = geojson.type;
72+
if (type !== 'Feature') {
73+
return Promise.reject(`feature ${i} has bad feature type: ${type}`);
74+
}
75+
76+
const geometry = geojson.geometry;
77+
if (!geometry) {
78+
return Promise.reject(`feature ${i} has no geometry`);
79+
}
80+
81+
let id;
82+
if (placeType === 'locality') {
83+
id = geojson.properties.CPOB_CCNCT;
84+
} else if (placeType === 'county') {
85+
id = `${geojson.properties.DPTO_CCDGO}${geojson.properties.MPIO_CCDGO}`;
86+
} else if (placeType === 'region') {
87+
id = geojson.properties.DPTO_CCDGO;
88+
}
89+
if (!id) {
90+
return Promise.reject(`feature ${i} has no id`);
91+
}
92+
93+
const name = geojson.properties.CPOB_CNMBR || geojson.properties.MPIO_CNMBR || geojson.properties.DPTO_CNMBR;
94+
if (!name) {
95+
return Promise.reject(`feature ${i} has no name`);
96+
}
97+
98+
const feature = {
99+
id: `divipola-${id}`,
100+
name: name,
101+
layer: placeType,
102+
hull: geometry,
103+
properties: {
104+
names: {
105+
DATA_LANGUAGE: name,
106+
},
107+
tags: [
108+
'boundary:administrative',
109+
`placetype:${placeType}`
110+
]
111+
}
112+
};
113+
114+
return new Promise((resolve, reject) => {
115+
services.features.upsert(feature, (err) => {
116+
if (err) return reject(err);
117+
118+
console.log(`done with feature ${i}: ${feature.id},${feature.name}`);
119+
resolve();
120+
});
121+
});
122+
}
123+
124+
function processFile(file) {
125+
return new Promise((resolve, reject) => {
126+
fs.readFile(file, (err, data) => {
127+
if (err) return reject(`error reading file: ${err}`);
128+
129+
let featureCollection;
130+
try {
131+
featureCollection = JSON.parse(data);
132+
} catch(e) {
133+
return reject(`failed to parse: ${file}`);
134+
}
135+
136+
const type = featureCollection.type;
137+
if (type !== 'FeatureCollection') {
138+
return reject(`not a feature collection: ${file}`);
139+
}
140+
141+
const features = featureCollection.features;
142+
if (!features || !features.length) {
143+
return reject(`no features in collection: ${file}`);
144+
}
145+
146+
let placeType;
147+
if (file.indexOf('MGN_RUR_CENTRO_POBLADO') !== -1) {
148+
placeType = 'locality';
149+
} else if (file.indexOf('MGN_ADM_MPIO_GRAFICO') !== -1) {
150+
placeType = 'county';
151+
} else if (file.indexOf('MGN_ADM_DPTO_POLITICO') !== -1) {
152+
placeType = 'region';
153+
} else {
154+
return reject(`unable to determine placeType for ${file}`);
155+
}
156+
157+
Promise.all(features.map((feature, i) => processFeature(placeType, feature, i)))
158+
.then(resolve)
159+
.catch(console.log);
160+
});
161+
});
162+
}
163+
164+
function processDirectory(dir) {
165+
return new Promise((resolve, reject) => {
166+
fs.readdir(dir, (err, files) => {
167+
if (err) return reject(err);
168+
169+
Promise.all(files.map(file => processFile(path.resolve(dir, file))))
170+
.then(resolve)
171+
.catch(console.log);
172+
});
173+
});
174+
};
175+
176+
services.init(err => {
177+
if (err) {
178+
console.log(err);
179+
process.exit(1);
180+
}
181+
182+
processDirectory('./divipola/geojson')
183+
.then(() => {
184+
console.log('all done');
185+
process.exit(0);
186+
})
187+
.catch((err) => {
188+
console.log(`finished with error: ${err}`);
189+
process.exit(1);
190+
});
191+
});

0 commit comments

Comments
 (0)