Upload replication files to azure and update the var for the code

Author: Rub21

envs/.env.db-utils.example

@@ -19,7 +19,7 @@ OVERWRITE_PLANET_FILE=false
 #######################################
 
 # TODO:update start.sh and helm templates, for making this customizable
-REPLICATION_FOLDER=/replication/minute
+REPLICATION_FOLDER=replication/minute
 
 #######################################
 # Environment variables for minute Replication job settings

images/replication-job/start.sh

@@ -8,33 +8,47 @@ else
 	echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
 fi
 
-workingDirectory="/mnt/data/data"
+workingDirectory="/mnt/data"
+mkdir -p $workingDirectory
 
 # Check if state.txt exist in the workingDirectory,
 # in case the file does not exist locally and does not exist in the cloud the replication will start from 0
 if [ ! -f $workingDirectory/state.txt ]; then
 	echo "File $workingDirectory/state.txt does not exist"
-	echo "let's get check the cloud"
+	### AWS
 	if [ $CLOUDPROVIDER == "aws" ]; then
-		aws s3 ls $AWS_S3_BUCKET$REPLICATION_FOLDER/state.txt
+		aws s3 ls $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt
 		if [[ $? -eq 0 ]]; then
-			echo "File exist, let's get it"
-			aws s3 cp $AWS_S3_BUCKET$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt
+			echo "File exist, let's get it from $CLOUDPROVIDER - $AWS_S3_BUCKET"
+			aws s3 cp $AWS_S3_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt
 		fi
 	fi
+
+	### GCP
 	if [ $CLOUDPROVIDER == "gcp" ]; then
-		gsutil ls $GCP_STORAGE_BUCKET$REPLICATION_FOLDER/state.txt
+		gsutil ls $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt
 		if [[ $? -eq 0 ]]; then
-			echo "File exist, let's get it"
-			gsutil cp $GCP_STORAGE_BUCKET$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt
+			echo "File exist, let's get it from $CLOUDPROVIDER - $GCP_STORAGE_BUCKET"
+			gsutil cp $GCP_STORAGE_BUCKET/$REPLICATION_FOLDER/state.txt $workingDirectory/state.txt
 		fi
 	fi
-	# In case the state.txt does not exist anywhere, lets create the folder data.
-	 mkdir -p $workingDirectory
+
+	### Azure
+	if [ $CLOUDPROVIDER == "azure" ]; then
+		state_file_exists=$(az storage blob exists --container-name $AZURE_CONTAINER_NAME --name $REPLICATION_FOLDER/state.txt --query="exists")
+		if [[ $state_file_exists=="true" ]]; then
+			echo "File exist, let's get it from $CLOUDPROVIDER - $AZURE_CONTAINER_NAME"
+			az storage blob download \
+				--container-name $AZURE_CONTAINER_NAME \
+				--name $REPLICATION_FOLDER/state.txt \
+				--file $workingDirectory/state.txt --query="name"
+		fi
+	fi
+	mkdir -p $workingDirectory
 fi
 
 # Creating the replication files
-function generateReplication () {
+function generateReplication() {
 	osmosis -q \
 		--replicate-apidb \
 		iterations=0 \
@@ -48,17 +62,31 @@ function generateReplication () {
 		--write-replication \
 		workingDirectory=$workingDirectory &
 	while true; do
-		for file in $(find $workingDirectory/ -cmin -1); do
-			if [ -f "$file" ]; then
-				bucketFile=${file#*"$workingDirectory"}
-				echo $(date +%F_%H:%M:%S)": New files..." $file
-				# AWS
+		for local_file in $(find $workingDirectory/ -cmin -1); do
+			if [ -f "$local_file" ]; then
+
+				cloud_file=$REPLICATION_FOLDER/${local_file#*"$workingDirectory/"}
+				echo $(date +%F_%H:%M:%S)": Copy file...$local_file to $cloud_file"
+
+				### AWS
 				if [ $CLOUDPROVIDER == "aws" ]; then
-					aws s3 cp $file $AWS_S3_BUCKET$REPLICATION_FOLDER$bucketFile --acl public-read
+					aws s3 cp $local_file $AWS_S3_BUCKET/$cloud_file --acl public-read
 				fi
-				# Google Storage
+
+				### GCP
 				if [ $CLOUDPROVIDER == "gcp" ]; then
-					gsutil cp -a public-read $file $GCP_STORAGE_BUCKET$REPLICATION_FOLDER$bucketFile
+					#TODO, emable public acces
+					gsutil cp -a public-read $local_file $GCP_STORAGE_BUCKET/$cloud_file
+				fi
+
+				### Azure
+				if [ $CLOUDPROVIDER == "azure" ]; then
+					#TODO, emable public acces
+					az storage blob upload \
+						--container-name $AZURE_CONTAINER_NAME \
+						--file $local_file \
+						--name $cloud_file \
+						--output none
 				fi
 			fi
 		done
@@ -69,8 +97,8 @@ function generateReplication () {
 # Check if Postgres is ready
 flag=true
 while "$flag" = true; do
-    pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
-        # Change flag to false to stop ping the DB
-        flag=false
-        generateReplication
-done
+	pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
+	# Change flag to false to stop ping the DB
+	flag=false
+	generateReplication
+done

osm-seed/templates/replication-job-deployment.yaml

@@ -41,7 +41,7 @@ spec:
             - name: POSTGRES_USER
               value: {{ .Values.db.env.POSTGRES_USER }}
             - name: REPLICATION_FOLDER
-              value: /replication/minute
+              value: replication/minute
             - name: CLOUDPROVIDER
               value: {{ .Values.cloudProvider }}
             # In case cloudProvider=aws