Add env var file and update container and readme for full-history

Author: Rub21

.env.full-history.example

@@ -0,0 +1,8 @@
+#######################################
+# Environment variables for overwrite full history planet PBF settings
+#######################################
+# OVERWRITE_FHISTORY_FILE=(true, false)
+OVERWRITE_FHISTORY_FILE=false
+
+#Osmosis tuning for server, This variable intends to use much as possible the memory in the server
+MEMORY_JAVACMD_OPTIONS=2Gi

docker-compose.yml

@@ -27,6 +27,14 @@ services:
     depends_on:
       - db
   # ######################################################
+  # ### OSM processor and also use this image as base for other containers
+  # ######################################################
+  osm-processor:
+    image: osmseed-osm-processor:v1
+    build:
+      context: ./images/osm-processor
+      dockerfile: Dockerfile
+  # ######################################################
   # ### Planet replication section
   # ######################################################
   planet-dump:
@@ -92,6 +100,28 @@ services:
       - ./.env.cloudprovider
     depends_on:
       - db
+  ######################################################
+  ### OSM full planet replication
+  ######################################################
+  full-history:
+    image: osmseed-full-history:v1
+    build:
+      context: ./images/full-history
+      dockerfile: Dockerfile
+    volumes:
+      - ./data/full-history-data:/mnt/data
+    command: >
+      /bin/bash -c "
+        echo 'Set cron job for full history PBF file, every 3m minutes';
+        sleep 1m;
+        while :; do echo 'Creating full history PBF file...'; /start.sh; sleep 3m; done;
+      "
+    env_file:
+      - ./.env.db
+      - ./.env.full-history
+      - ./.env.cloudprovider
+    depends_on:
+      - db
   # ######################################################
   # ### OSM data restoring into the database
   # ######################################################
@@ -101,7 +131,7 @@ services:
   #     context: ./images/populate-apidb
   #     dockerfile: Dockerfile
   #   volumes:
-  #     - ./populate-apidb-data:/app/data
+  #     - ./data/populate-apidb-data:/app/data
   #   depends_on:
   #     - db
   #   command: >
@@ -112,34 +142,6 @@ services:
   #     ./start.sh"
   #   env_file:
   #     - ./.env
-  # osm-processor:
-  #   image: osmseed-osm-processor:v1
-  #   build:
-  #     context: ./images/osm-processor
-  #     dockerfile: Dockerfile
-  #   env_file:
-  #     - ./.env
-  # ######################################################
-  # ### OSM full planet replication
-  # ######################################################
-  # full-history:
-  #   image: osmseed-full-history:v1
-  #   build:
-  #     context: ./images/full-history
-  #     dockerfile: Dockerfile
-  #   depends_on:
-  #     - web
-  #     - db
-  #   volumes:
-  #     - ./full-history-data:/mnt/data
-  #   command: >
-  #     /bin/bash -c "
-  #     echo Sleep full-history for a while!;
-  #     sleep 40;
-  #     echo Starting full-history!;
-  #     ./start.sh"
-  #   env_file:
-  #     - ./.env
   # ######################################################
   # ### Tiler server(tegola), Database and  imposm section
   # ######################################################

images/full-history/Dockerfile

@@ -1,4 +1,5 @@
 FROM developmentseed/osmseed-osm-processor:0.1.0-n487.hbce9469
-WORKDIR /mnt/data
-COPY ./start.sh .
-CMD ./start.sh
+
+VOLUME /mnt/data
+COPY ./start.sh /
+CMD /start.sh

images/full-history/README.md

@@ -2,46 +2,28 @@
 
 Dockerfile for getting the full planet history of database.
 
-
 ### Configuration
 
-Required environment variables;
-
-- `POSTGRES_HOST`
-- `POSTGRES_DB`
-- `POSTGRES_USER`
-- `POSTGRES_PASSWORD`
-
-The following env variables are according to which cloud provider you are going to use:
-
-- `CLOUDPROVIDER`, eg. `aws` or `gcp`
-
-In case AWS:
-
-- `AWS_S3_BUCKET` e.g `s3://osm-seed-test`
+In order to run this container we need environment variables, these can be found in the following files👇:
 
-In case GCP:
+- [.env.db.example](./../../.env.db.example)
+- [.env.full-history.example](./../../.env.full-history.example)
+- [.env.cloudprovider.example](./../../.env.cloudprovider.example)
 
-- `GCP_STORAGE_BUCKET` e.g `gs://osm-seed-test`
+**Note**: Rename the above files as `.env.db`, `.env.full-history` and `.env.cloudprovider`
 
-By default, the container create a new *.osm.bz2 file, if you want to overwrite the same file as `history-latest.osm.bz2` set the following env var 👇;
-
-- `OVERWRITE_FHISTORY_FILE=yes`
-
-
-#### Building the container
+#### Running full-history container
 
 ```sh
-    cd full-history
-    docker build -t full-history:v1 .
+    # Docker compose
+    docker-compose run full-history
+
+    # Docker
+    docker run \
+    --env-file ./../.env.db \
+    --env-file ./../.env.full-history \
+    --env-file ./../.env.cloudprovider \
+    -v ${PWD}/data/full-history-data:/mnt/data \
+    --network osm-seed_default \
+    -it osmseed-full-history:v1
 ```
-
-#### Running the container
-
-```sh
-docker run \
---env-file ./../.env \
--v ${PWD}:/mnt/data \
---network osm-seed_default \
--it full-history:v1
-```

images/full-history/start.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+export VOLUME_DIR=/mnt/data
 
 # osmosis tuning: https://wiki.openstreetmap.org/wiki/Osmosis/Tuning,https://lists.openstreetmap.org/pipermail/talk/2012-October/064771.html
 if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then
@@ -10,14 +11,14 @@ fi
 
 # Fixing name for historical file
 date=$(date '+%y%m%d_%H%M')
-fullHistoryFile=history-${date}.osh.pbf
+fullHistoryFile=$VOLUME_DIR/history-${date}.osh.pbf
 # In case overwrite the file
 if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
-	fullHistoryFile=history-latest.osh.pbf
+	fullHistoryFile=$VOLUME_DIR/history-latest.osh.pbf
 fi
 
 # State file nname
-stateFile="state.txt"
+stateFile="$VOLUME_DIR/state.txt"
 osm_tmp_file="osm_tmp.osm"
 
 # Creating full history
@@ -36,6 +37,9 @@ osmosis --read-apidb-change \
 osmium cat $osm_tmp_file -o $fullHistoryFile
 osmium fileinfo $fullHistoryFile
 
+# Remove full-hitory osm file, keep only history-latest.osh.pbf files
+rm $osm_tmp_file
+
 # AWS
 if [ $CLOUDPROVIDER == "aws" ]; then
 	AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
@@ -45,7 +49,7 @@ if [ $CLOUDPROVIDER == "aws" ]; then
 	aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
 fi
 
-# Google Storage
+# Google Storagegit status
 if [ $CLOUDPROVIDER == "gcp" ]; then
 	echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" >$stateFile
 	# Upload to GS

osm-seed/templates/full-history-job.yaml

@@ -21,7 +21,7 @@ spec:
           containers:
           - name: {{ .Release.Name }}-full-history-job
             image: {{ .Values.fullHistory.image.name }}:{{ .Values.fullHistory.image.tag }}
-            command: ['./start.sh']
+            command: ['/start.sh']
             {{- if .Values.fullHistory.resources.enabled }}   
             resources:
               requests: