Skip to content

Commit 3f05888

Browse files
ZeLonewolfZeLonewolf
authored
Fix download-osm and remove osmborder (#462)
This PR improves error handling in the download-osm script by replacing the deprecated asyncio.wait() function with asyncio.gather() and adding proper exception handling. It also updates the catalog search code and makes the Docker container work again. Additionally, this PR completely removes osmborder, which is no longer used after openmaptiles/openmaptiles#1213
1 parent a4ba8bc commit 3f05888

24 files changed

+187
-355
lines changed

‎Dockerfile‎

Lines changed: 46 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.17 as go-builder
1+
FROM golang:1.17 AS go-builder
22
ARG IMPOSM_REPO="https://github.com/omniscale/imposm3.git"
33
ARG IMPOSM_VERSION="v0.11.1"
44

@@ -30,36 +30,8 @@ RUN set -eux ;\
3030
( [ -f imposm ] && mv imposm /build-bin/imposm || mv imposm3 /build-bin/imposm )
3131

3232

33-
# Build osmborder
34-
FROM python:3.9 as c-builder
35-
ARG OSMBORDER_REV=e3ae8f7a2dcdcd6dc80abab4679cb5edb7dc6fa5
36-
37-
RUN set -eux ;\
38-
mkdir /build-bin ;\
39-
DEBIAN_FRONTEND=noninteractive apt-get update ;\
40-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
41-
`# installing osmborder dependencies` \
42-
build-essential \
43-
ca-certificates \
44-
cmake \
45-
git \
46-
libosmium2-dev \
47-
zlib1g-dev \
48-
;\
49-
/bin/bash -c 'echo ""; echo ""; echo "##### Building osmborder -- https://github.com/pnorman/osmborder"' >&2 ;\
50-
git clone https://github.com/pnorman/osmborder.git /usr/src/osmborder ;\
51-
cd /usr/src/osmborder ;\
52-
git checkout ${OSMBORDER_REV:?} ;\
53-
mkdir -p /usr/src/osmborder/build ;\
54-
cd /usr/src/osmborder/build ;\
55-
cmake .. ;\
56-
make ;\
57-
make install ;\
58-
mv /usr/src/osmborder/build/src/osmborder /build-bin ;\
59-
mv /usr/src/osmborder/build/src/osmborder_filter /build-bin
60-
6133
# Build SPREET
62-
FROM rust:1.76 as rust-builder
34+
FROM rust:1.76 AS rust-builder
6335
ARG SPREET_REPO="https://github.com/flother/spreet"
6436
ARG SPREET_VERSION="v0.11.0"
6537

@@ -78,15 +50,16 @@ LABEL maintainer="Yuri Astrakhan <YuriAstrakhan@gmail.com>"
7850
ARG PG_MAJOR=14
7951
ARG TOOLS_DIR=/usr/src/app
8052

81-
WORKDIR ${TOOLS_DIR}
53+
WORKDIR /tileset
8254

8355
#
8456
# IMPOSM_CONFIG_FILE can be used to provide custom IMPOSM config file
8557
# SQL_TOOLS_DIR can be used to provide custom SQL files instead of the files from /sql
8658
#
8759
ENV TOOLS_DIR="$TOOLS_DIR" \
88-
PATH="${TOOLS_DIR}:${PATH}" \
89-
IMPOSM_CONFIG_FILE=${TOOLS_DIR}/config/repl_config.json \
60+
PATH="${TOOLS_DIR}/bin:${PATH}" \
61+
PYTHONPATH="${TOOLS_DIR}:${PYTHONPATH}" \
62+
IMPOSM_CONFIG_FILE=${TOOLS_DIR}/bin/config/repl_config.json \
9063
IMPOSM_MAPPING_FILE=/mapping/mapping.yaml \
9164
IMPOSM_CACHE_DIR=/cache \
9265
IMPOSM_DIFF_DIR=/import \
@@ -109,63 +82,55 @@ RUN set -eux ;\
10982
nano \
11083
procps `# ps command` \
11184
gnupg2 `# TODO: not sure why gnupg2 is needed` ;\
112-
curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - ;\
113-
/bin/bash -c 'source /etc/os-release && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${VERSION_CODENAME:?}-pgdg main ${PG_MAJOR:?}" > /etc/apt/sources.list.d/pgdg.list' ;\
85+
curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg ;\
86+
/bin/bash -c 'source /etc/os-release && echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt/ ${VERSION_CODENAME:?}-pgdg main ${PG_MAJOR:?}" > /etc/apt/sources.list.d/pgdg.list' ;\
11487
DEBIAN_FRONTEND=noninteractive apt-get update ;\
11588
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
11689
aria2 `# multi-stream file downloader - used by download-osm` \
11790
graphviz `# used by layer mapping graphs` \
118-
sqlite3 `# mbtiles file manipulations` \
119-
gdal-bin `# contains ogr2ogr` \
120-
osmctools `# osmconvert and other OSM tools` \
121-
osmosis `# useful toolset - https://wiki.openstreetmap.org/wiki/Osmosis` \
122-
postgresql-client-${PG_MAJOR:?} `# psql` \
123-
\
124-
# imposm dependencies
125-
libgeos-dev \
126-
libleveldb-dev \
127-
libprotobuf-dev \
91+
sqlite3 `# mbtiles file manipulations` \
92+
postgresql-client-${PG_MAJOR:?} \
93+
postgresql-${PG_MAJOR:?}-postgis-3 \
94+
osmium-tool \
95+
osmctools `# contains osmconvert` \
96+
libgeos-dev `# Imposm dependency` \
97+
libleveldb-dev `# Imposm dependency` \
98+
libprotobuf-dev `# Imposm dependency` \
99+
unzip \
128100
;\
129-
# generate-tiles
130-
curl -sL https://deb.nodesource.com/setup_14.x | bash - ;\
131-
DEBIAN_FRONTEND=noninteractive apt-get update ;\
132-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
133-
nodejs npm build-essential ;\
134-
rm -rf /var/lib/apt/lists/ ;\
135-
npm install -g \
136-
@mapbox/mbtiles@0.12.1 \
137-
@mapbox/tilelive@6.1.1 \
138-
tilelive-pgquery@1.2.0 ;\
139-
\
140-
/bin/bash -c 'echo ""; echo ""; echo "##### Cleaning up"' >&2 ;\
101+
apt-get clean ;\
141102
rm -rf /var/lib/apt/lists/*
142103

143-
RUN groupadd --gid 1000 openmaptiles \
144-
&& useradd --uid 1000 --gid openmaptiles --shell /bin/bash --create-home openmaptiles
104+
# Copy built binaries from build stages
105+
COPY --from=go-builder /build-bin/imposm /usr/local/bin/
106+
COPY --from=rust-builder /build-bin/spreet /usr/local/bin/
145107

146-
# Copy requirements.txt first to avoid pip install on every code change
147-
COPY ./requirements.txt .
148-
RUN pip install --no-cache-dir -r requirements.txt
108+
# Copy Python tools
109+
COPY . ${TOOLS_DIR}/
149110

150-
# Copy tools, imposm, osmborder and spreet into the app dir
151-
COPY --from=go-builder /build-bin/* ./
152-
COPY --from=c-builder /build-bin/* ./
153-
COPY --from=rust-builder /build-bin/* ./
154-
COPY . .
111+
# Copy config file to expected location
112+
RUN mkdir -p ${TOOLS_DIR}/config && \
113+
cp ${TOOLS_DIR}/bin/config/repl_config.json ${TOOLS_DIR}/config/repl_config.json
155114

156-
RUN set -eux ;\
157-
mv bin/* . ;\
158-
rm -rf bin ;\
159-
rm requirements.txt ;\
160-
./download-osm list geofabrik ;\
161-
./download-osm list bbbike
115+
# Install Python dependencies
116+
RUN pip install --no-cache-dir -r ${TOOLS_DIR}/requirements.txt
162117

163-
WORKDIR /tileset
118+
# Install Node.js 18 and essential tilelive packages
119+
RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
120+
apt-get update && \
121+
apt-get install -y nodejs build-essential python3-dev libsqlite3-dev && \
122+
npm install -g \
123+
@mapbox/mbtiles@0.11.0 \
124+
@mapbox/tilelive@6.1.0 \
125+
nyurik/tilelive-pgquery \
126+
--unsafe-perm
127+
128+
# Create necessary directories
129+
RUN mkdir -p /cache /import /mapping /usr/src/app/data /usr/src/app/build/openmaptiles.tm2source
130+
131+
# Set proper permissions
132+
RUN chmod +x ${TOOLS_DIR}/bin/*
133+
RUN chmod 777 /cache /import /mapping /usr/src/app/data /usr/src/app/build/openmaptiles.tm2source
164134

165-
# In case there are no parameters, print a list of available scripts
166-
CMD echo "*******************************************************************" ;\
167-
echo " Please specify a script to run. Here are the available scripts." ;\
168-
echo " Use script name with --help to get more information." ;\
169-
echo " Use 'bash' to start a shell inside the tools container." ;\
170-
echo "*******************************************************************" ;\
171-
find "${TOOLS_DIR}" -maxdepth 1 -executable -type f -printf " * %f\n" | sort
135+
# Default command
136+
CMD ["/bin/bash"]

‎Makefile‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,9 +107,18 @@ run-python-tests: build-docker
107107
@echo " Running Python unit tests"
108108
@echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
109109
@$(RUN_CMD) $(DOCKER_IMAGE) bash -c \
110-
'python -m flake8 openmaptiles tests/python `grep -rIzl "^#!.*python" bin` \
111-
&& (python -m unittest discover 2>&1 | \
112-
awk -v s="Ran 0 tests in" '\''$$0~s{print; print "\n*** No Python unit tests found, aborting"; exit(1)} 1'\'')'
110+
'cd /usr/src/app && \
111+
if [ -d "tests/python" ]; then \
112+
python -m flake8 openmaptiles tests/python `grep -rIzl "^#!.*python" bin`; \
113+
else \
114+
python -m flake8 openmaptiles `grep -rIzl "^#!.*python" bin`; \
115+
fi && \
116+
if [ -d "tests/python" ]; then \
117+
cd tests/python && python -m unittest discover -p "test_*.py" 2>&1 | \
118+
awk -v s="Ran 0 tests in" '\''$$0~s{print; print "\n*** No Python unit tests found, aborting"; exit(1)} 1'\''; \
119+
else \
120+
echo "No Python tests found, skipping"; \
121+
fi'
113122

114123
.PHONY: build-sql-tests
115124
build-sql-tests: prepare build-docker
@@ -119,7 +128,7 @@ build-sql-tests: prepare build-docker
119128
@echo " Running Postgres SQL tests"
120129
@echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
121130
$(DOCKER_COMPOSE_COMMAND) --file tests/sql/docker-compose.yml rm -f && \
122-
$(DOCKER_COMPOSE_COMMAND) --file tests/sql/docker-compose.yml up --abort-on-container-exit --wait-timeout 180 && \
131+
$(DOCKER_COMPOSE_COMMAND) --file tests/sql/docker-compose.yml up --abort-on-container-exit --timeout 180 && \
123132
$(DOCKER_COMPOSE_COMMAND) --file tests/sql/docker-compose.yml rm -f
124133

125134
.PHONY: build-bin-tests
@@ -128,8 +137,8 @@ build-bin-tests: prepare build-docker
128137
@echo " Running tools integration tests"
129138
@echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
130139
$(RUN_CMD) -e "BUILD=/tileset/$(BUILD_DIR)" \
131-
-v "$(WORKDIR)/tests/cache:/usr/src/app/cache" \
132-
$(DOCKER_IMAGE) tests/test-tools.sh
140+
-v "$(WORKDIR)/tests/cache:/cache" \
141+
$(DOCKER_IMAGE) /tileset/tests/test-tools.sh
133142

134143
.PHONY: build-tests
135144
build-tests: build-bin-tests build-sql-tests

‎README.md‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -428,19 +428,6 @@ You need to provide PostgreSQL database connection settings before generating th
428428
generate-tm2source <tileset> --host="localhost" --port=5432 --database="osm" --user="osm" --password="osm"
429429
```
430430

431-
### Import OSM Borders
432-
433-
The **import-borders** script will take the first PBF file from the `/import` dir (by default), extract borders with [osmborder tool](https://github.com/pnorman/osmborder), and import resulting CSV file into the database as osm_border_linestring table (by default). Example usages:
434-
435-
```bash
436-
import-borders # Parse and import first PBF file in PBF_DATA_DIR
437-
import-borders [import] planet.pbf # Parse and import planet.pbf
438-
import-borders parse planet.pbf # Parse planet.pbf into a CSV file, but do not import
439-
import-borders load borders.csv # Load borders.csv into a table
440-
```
441-
442-
This utility requires PostgreSQL's PG* environment variables, and optionally uses `PBF_DATA_DIR, BORDERS_PBF_FILE, BORDERS_CSV_FILE, BORDERS_TABLE_NAME`.
443-
444431
## Importing into Postgres
445432
The `import-sql` script can execute a single SQL file in Postgres when the file is given as the first parameter.
446433

0 commit comments

Comments
 (0)