Skip to content

Commit 7144c8c

Browse files
author
Rub21
committed
Azure support for planet-dump and full history
1 parent 57bdcef commit 7144c8c

File tree

6 files changed

+94
-26
lines changed

6 files changed

+94
-26
lines changed

‎compose/full-history.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ services:
1313
command: >
1414
/bin/bash -c "
1515
echo 'Set cron job for full history PBF file, every 4 minutes';
16-
while :; do sleep 4m; echo 'Creating full history PBF file...'; /start.sh; done;
16+
while :; do echo 'Creating full history PBF file...'; /start.sh; sleep 4m; done;
1717
"
1818
env_file:
1919
- ../envs/.env.db

‎compose/planet-dump.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ services:
2121
command: >
2222
/bin/bash -c "
2323
echo 'Set cronjob for planet-dump, every 4 minutes';
24-
while :; do sleep 4m; echo 'Creating the planet dump file...'; /start.sh; done;
24+
while :; do echo 'Creating the planet dump file...'; /start.sh; sleep 4m; done;
2525
"
2626
env_file:
2727
- ../envs/.env.db

‎images/full-history/start.sh

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@ fi
1111

1212
# Fixing name for historical file
1313
date=$(date '+%y%m%d_%H%M')
14-
fullHistoryFile=$VOLUME_DIR/history-${date}.osh.pbf
14+
local_fullHistoryFile=$VOLUME_DIR/history-${date}.osh.pbf
15+
cloud_fullHistoryFile=planet/full-history/history-${date}.osh.pbf
16+
1517
# In case overwrite the file
1618
if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
17-
fullHistoryFile=$VOLUME_DIR/history-latest.osh.pbf
19+
local_fullHistoryFile=$VOLUME_DIR/history-latest.osh.pbf
20+
cloud_fullHistoryFile=planet/full-history/history-latest.osh.pbf
1821
fi
1922

2023
# State file nname
@@ -34,25 +37,45 @@ osmosis --read-apidb-change \
3437
$osm_tmp_file
3538

3639
# Convert file to PBF file
37-
osmium cat $osm_tmp_file -o $fullHistoryFile
38-
osmium fileinfo $fullHistoryFile
40+
osmium cat $osm_tmp_file -o $local_fullHistoryFile
41+
osmium fileinfo $local_fullHistoryFile
3942

4043
# Remove full-hitory osm file, keep only history-latest.osh.pbf files
4144
rm $osm_tmp_file
4245

4346
# AWS
4447
if [ $CLOUDPROVIDER == "aws" ]; then
4548
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
46-
echo "$AWS_URL.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile
47-
# Upload to S3
48-
aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
49-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
49+
echo "$AWS_URL.s3.amazonaws.com/$cloud_fullHistoryFile" >$stateFile
50+
# Upload history-planet.osm.pbf
51+
aws s3 cp $local_fullHistoryFile $AWS_S3_BUCKET/$cloud_fullHistoryFile --acl public-read
52+
# Upload state.txt
53+
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/state.txt --acl public-read
5054
fi
5155

5256
# Google Storage
5357
if [ $CLOUDPROVIDER == "gcp" ]; then
54-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" >$stateFile
55-
# Upload to GS
56-
gsutil cp -a public-read $fullHistoryFile $GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile
57-
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/$stateFile
58+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_fullHistoryFile" >$stateFile
59+
# Upload history-planet.osm.pbf
60+
gsutil cp -a public-read $local_fullHistoryFile $GCP_STORAGE_BUCKET/$cloud_fullHistoryFile
61+
# Upload state.txt
62+
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/state.txt
63+
fi
64+
65+
# Azure
66+
if [ $CLOUDPROVIDER == "azure" ]; then
67+
# Save the path file
68+
echo "https://$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$AZURE_CONTAINER_NAME/$cloud_fullHistoryFile" >$stateFile
69+
# Upload history-planet.osm.pbf
70+
az storage blob upload \
71+
--container-name $AZURE_CONTAINER_NAME \
72+
--file $local_fullHistoryFile \
73+
--name $cloud_fullHistoryFile \
74+
--output table
75+
# Upload state.txt
76+
az storage blob upload \
77+
--container-name $AZURE_CONTAINER_NAME \
78+
--file $stateFile \
79+
--name planet/full-history/state.txt \
80+
--output table
5881
fi

‎images/planet-dump/start.sh

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@ fi
1111

1212
# Read the DB and create the planet osm file
1313
date=$(date '+%y%m%d_%H%M')
14-
planetPBFFile=$VOLUME_DIR/planet-${date}.osm.pbf
14+
local_planetPBFFile=$VOLUME_DIR/planet-${date}.osm.pbf
15+
cloud_planetPBFFile=planet/planet-${date}.osm.pbf
16+
1517
# In case overwrite the file
1618
if [ "$OVERWRITE_PLANET_FILE" == "true" ]; then
17-
planetPBFFile=$VOLUME_DIR/planet-latest.osm.pbf
19+
local_planetPBFFile=$VOLUME_DIR/planet-latest.osm.pbf
20+
cloud_planetPBFFile=planet/planet-latest.osm.pbf
1821
fi
1922

2023
stateFile="$VOLUME_DIR/state.txt"
@@ -27,23 +30,43 @@ osmosis --read-apidb \
2730
password=$POSTGRES_PASSWORD \
2831
validateSchemaVersion=no \
2932
--write-pbf \
30-
file=$planetPBFFile
33+
file=$local_planetPBFFile
3134

3235
# AWS
3336
if [ $CLOUDPROVIDER == "aws" ]; then
3437
# Save the path file
3538
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
36-
echo "$AWS_URL.s3.amazonaws.com/planet/$planetPBFFile" > $stateFile
37-
# Upload to S3
38-
aws s3 cp $planetPBFFile $AWS_S3_BUCKET/planet/$planetPBFFile --acl public-read
39-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/$stateFile --acl public-read
39+
echo "$AWS_URL.s3.amazonaws.com/planet/$local_planetPBFFile" > $stateFile
40+
# Upload planet.osm.pbf file to s3
41+
aws s3 cp $local_planetPBFFile $AWS_S3_BUCKET/$cloud_planetPBFFile --acl public-read
42+
# Upload state.txt file to s3
43+
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/state.txt --acl public-read
4044
fi
4145

42-
# Google Storage
46+
# gcp
4347
if [ $CLOUDPROVIDER == "gcp" ]; then
4448
# Save the path file
45-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/$planetPBFFile" > $stateFile
46-
# Upload to GS
47-
gsutil cp -a public-read $planetPBFFile $GCP_STORAGE_BUCKET/planet/$planetPBFFile
48-
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/$stateFile
49+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/$cloud_planetPBFFile" > $stateFile
50+
# Upload planet.osm.pbf file to cloud storage
51+
gsutil cp -a public-read $local_planetPBFFile $GCP_STORAGE_BUCKET/$cloud_planetPBFFile
52+
# Upload state.txt file to cloud storage
53+
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/state.txt
54+
fi
55+
56+
# Azure
57+
if [ $CLOUDPROVIDER == "azure" ]; then
58+
# Save the path file
59+
echo "https://$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$AZURE_CONTAINER_NAME/$cloud_planetPBFFile" > $stateFile
60+
# Upload planet.osm.pbf file to blob storage
61+
az storage blob upload \
62+
--container-name $AZURE_CONTAINER_NAME \
63+
--file $local_planetPBFFile \
64+
--name $cloud_planetPBFFile \
65+
--output table
66+
# Upload state.txt file to blob storage
67+
az storage blob upload \
68+
--container-name $AZURE_CONTAINER_NAME \
69+
--file $stateFile \
70+
--name planet/state.txt \
71+
--output table
4972
fi

‎osm-seed/templates/full-history-job.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,17 @@ spec:
5454
- name: GCP_STORAGE_BUCKET
5555
value: {{ .Values.GCP_STORAGE_BUCKET }}
5656
{{- end }}
57+
# In case cloudProvider=azure
58+
{{- if eq .Values.cloudProvider "azure" }}
59+
- name: AZURE_STORAGE_ACCOUNT
60+
value: {{ .Values.AZURE_STORAGE_ACCOUNT }}
61+
- name: AZURE_CONTAINER_NAME
62+
value: {{ .Values.AZURE_CONTAINER_NAME }}
63+
- name: AZURE_STORAGE_ACCESS_KEY
64+
value: {{ .Values.AZURE_STORAGE_ACCESS_KEY }}
65+
- name: AZURE_STORAGE_CONNECTION_STRING
66+
value: {{ .Values.AZURE_STORAGE_CONNECTION_STRING }}
67+
{{- end }}
5768
# Memory optimization for osmosis
5869
{{- if .Values.fullHistory.resources.enabled }}
5970
- name: MEMORY_JAVACMD_OPTIONS

‎osm-seed/templates/planet-dump-job.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,17 @@ spec:
5454
- name: GCP_STORAGE_BUCKET
5555
value: {{ .Values.GCP_STORAGE_BUCKET }}
5656
{{- end }}
57+
# In case cloudProvider=azure
58+
{{- if eq .Values.cloudProvider "azure" }}
59+
- name: AZURE_STORAGE_ACCOUNT
60+
value: {{ .Values.AZURE_STORAGE_ACCOUNT }}
61+
- name: AZURE_CONTAINER_NAME
62+
value: {{ .Values.AZURE_CONTAINER_NAME }}
63+
- name: AZURE_STORAGE_ACCESS_KEY
64+
value: {{ .Values.AZURE_STORAGE_ACCESS_KEY }}
65+
- name: AZURE_STORAGE_CONNECTION_STRING
66+
value: {{ .Values.AZURE_STORAGE_CONNECTION_STRING }}
67+
{{- end }}
5768
# Memory optimization for osmosis
5869
{{- if .Values.planetDump.resources.enabled }}
5970
- name: MEMORY_JAVACMD_OPTIONS

0 commit comments

Comments
 (0)