|
| 1 | +--- |
| 2 | +apiVersion: v1 |
| 3 | +data: |
| 4 | + gpkg_download.sh: |- |
| 5 | + #!/usr/bin/env bash |
| 6 | +
|
| 7 | + set -euo pipefail |
| 8 | +
|
| 9 | + function download_gpkg() { |
| 10 | + local gpkg=$1 |
| 11 | + local file=$2 |
| 12 | + local url=$3 |
| 13 | +
|
| 14 | + if [ -f "$file" ] && [ ! -f "$file".st ]; then |
| 15 | + echo msg=\"File already downloaded\" file=\""$file"\" |
| 16 | + else |
| 17 | + echo msg=\"Starting download\" gpkg=\""$gpkg"\" file=\""$file"\" url=\""$url"\" |
| 18 | +
|
| 19 | + # use curl to check if resource exists |
| 20 | + # axel blocks on non-existing resources |
| 21 | + curl -IfsS "$url" > /dev/null |
| 22 | +
|
| 23 | + echo start "$gpkg" |
| 24 | + ret=0 |
| 25 | + # Connection timeout causes file corruption https://github.com/axel-download-accelerator/axel/issues/303, therefore we set the timeout to 2 minutes. |
| 26 | + axel -n 1 -T 120 -o "$file" "$url" \ |
| 27 | + | sed -r 's|\[[[:space:]]*(.*)%\].*\[(.*)/s\]|progress \1 \2|p' \ |
| 28 | + | sed -r 's|\[[[:space:]]*(.*)%\].*|progress \1|p' || ret=$? |
| 29 | +
|
| 30 | + if [ $ret -ne 0 ] |
| 31 | + then |
| 32 | + echo -e '\n' |
| 33 | + # Download failed ($? != 0). |
| 34 | + if [ $ret -eq 1 ] |
| 35 | + then |
| 36 | + # Axel was not able to resume ($? == 1). Remove file and state file. |
| 37 | + if [ -f "$file" ]; then |
| 38 | + echo msg=\"Resume failed, removing file\" file=\""$file"\" |
| 39 | + rm "$file" |
| 40 | + fi |
| 41 | + if [ -f "$file.st" ]; then |
| 42 | + echo msg=\"Resume failed, removing file\" file=\""$file".st\" |
| 43 | + rm "$file".st |
| 44 | + fi |
| 45 | + else |
| 46 | + # Download failed with other error ($? > 1). Remove file if state file does not exist. |
| 47 | + if [ ! -f "$file.st" ]; then |
| 48 | + echo msg=\"Download failed without state file, removing file\" file=\""$file"\" |
| 49 | + rm "$file" |
| 50 | + fi |
| 51 | + fi |
| 52 | +
|
| 53 | + # Retry the download |
| 54 | + echo msg=\"Retry file\" file=\""$file"\" |
| 55 | + download_gpkg $gpkg $file $url |
| 56 | + fi |
| 57 | + fi |
| 58 | + } |
| 59 | +
|
| 60 | + function download() { |
| 61 | + if [ -z "$BLOBS_ENDPOINT" ]; |
| 62 | + then |
| 63 | + echo echo "Empty BLOBS_ENDPOINT variable, start script with 'blobs' configmap"; |
| 64 | + exit 1; |
| 65 | + fi |
| 66 | +
|
| 67 | + local gpkg=$1 |
| 68 | + local file=/srv/data/gpkg/$2 |
| 69 | + local url=${BLOBS_ENDPOINT}/${gpkg} |
| 70 | +
|
| 71 | + download_gpkg $gpkg $file $url |
| 72 | +
|
| 73 | + # Check Content-length |
| 74 | + download_size=$(curl -sI "$url" | grep -i Content-Length | awk '{print $2}' | tr -d '\r') |
| 75 | + file_size=$(wc -c "$file" | awk '{print $1}') |
| 76 | + if [ "$download_size" != "$file_size" ] |
| 77 | + then |
| 78 | + echo msg=\"Content-length mismatch\" file=\""$file"\" file_size=\""$file_size"\" download_size=\""$download_size"\" |
| 79 | + rm_file_and_exit |
| 80 | + else |
| 81 | + echo msg=\"Content-length match\" file=\""$file"\" file_size=\""$file_size"\" download_size=\""$download_size"\" |
| 82 | + chown 999:999 "$file" |
| 83 | + fi |
| 84 | +
|
| 85 | + # Check ogrinfo |
| 86 | + echo "Check gpkg with ogrinfo" |
| 87 | + if ! ogrinfo -so "$file" |
| 88 | + then |
| 89 | + echo "ERROR: ogrinfo check on $file failed" |
| 90 | + rm_file_and_exit |
| 91 | + fi |
| 92 | +
|
| 93 | + # Only check md5 hash if a valid md5 value is returned (for large blobs Azure doesn't return a md5sum) |
| 94 | + echo "Check if md5 hash value exists in blob storage" |
| 95 | + rclone md5sum "blobs:${gpkg}" --output-file "${file}.md5sum-remote" |
| 96 | +
|
| 97 | + # If file contains valid hash, then check it, else skip |
| 98 | + hash=$(awk '{ print $1 }' "${file}.md5sum-remote") |
| 99 | + if [[ $hash =~ ^[a-f0-9]{32}$ ]] |
| 100 | + then |
| 101 | + echo "Valid hash value found" |
| 102 | + echo "Compare MD5 hash of remote and downloaded gpkg" |
| 103 | + if ! (echo "$hash $file" | md5sum --check); then |
| 104 | + rm_file_and_exit |
| 105 | + fi |
| 106 | + else |
| 107 | + echo "No hash found for $file in blob storage, skipping checksum." |
| 108 | + fi |
| 109 | +
|
| 110 | + echo "done" |
| 111 | + } |
| 112 | +
|
| 113 | + function download_all() { |
| 114 | + echo msg=\"Starting GeoPackage downloader\" |
| 115 | +
|
| 116 | + local start_time=$(date '+%s') |
| 117 | +
|
| 118 | + # create target location if not exists |
| 119 | + mkdir -p /srv/data/gpkg |
| 120 | + chown 999:999 /srv/data/gpkg |
| 121 | +
|
| 122 | + download ${BLOBS_GEOPACKAGES_BUCKET}/key/file.gpkg file.gpkg; |
| 123 | +
|
| 124 | + echo msg=\"All GeoPackages downloaded\" total_time_seconds=$(expr $(date '+%s') - $start_time) |
| 125 | + } |
| 126 | +
|
| 127 | + function rm_file_and_exit() { |
| 128 | + echo "Removing $file, to ensure a fresh new download is started when script is executed again" |
| 129 | + rm -rf "$file" |
| 130 | +
|
| 131 | + if [ -f "$file.st" ]; then |
| 132 | + rm "$file".st |
| 133 | + fi |
| 134 | +
|
| 135 | + echo "Exiting..." |
| 136 | + exit 1 |
| 137 | + } |
| 138 | +
|
| 139 | + download_all | awk -W interactive ' |
| 140 | + BEGIN { |
| 141 | + state="idle"; |
| 142 | + } |
| 143 | +
|
| 144 | + { |
| 145 | + if ($0 != "") { |
| 146 | + if ($1 == "start") { |
| 147 | + gpkg=$2; |
| 148 | + state="downloading"; |
| 149 | + } else if ($1 == "done") { |
| 150 | + state="idle"; |
| 151 | + } else if (state == "downloading") { |
| 152 | + if ($1 == "progress") { |
| 153 | + # reduce output to prevent loki from choking on large log volume |
| 154 | + if (last_percentage != $2) { |
| 155 | + if ($3 == "") { |
| 156 | + print "msg=\"Downloading\" gpkg=" gpkg " percentage=" $2; |
| 157 | + } else { |
| 158 | + print "msg=\"Downloading\" gpkg=" gpkg " percentage=" $2 " bytes_per_second=" $3; |
| 159 | + } |
| 160 | + } |
| 161 | + last_percentage=$2; |
| 162 | + } else { |
| 163 | + print "msg=\"" $0 "\" gpkg=" gpkg; |
| 164 | + } |
| 165 | + } else { |
| 166 | + print $0; |
| 167 | + } |
| 168 | + } |
| 169 | + } |
| 170 | + ' |
| 171 | +immutable: true |
| 172 | +kind: ConfigMap |
| 173 | +metadata: |
| 174 | + labels: |
| 175 | + app: mapserver |
| 176 | + dataset: dataset |
| 177 | + dataset-owner: datasetOwner |
| 178 | + inspire: "false" |
| 179 | + service-type: wms |
| 180 | + service-version: v1_0 |
| 181 | + name: patches-wms-init-scripts-fft29bbtdd |
| 182 | + namespace: default |
| 183 | + ownerReferences: |
| 184 | + - apiVersion: pdok.nl/v3 |
| 185 | + kind: WMS |
| 186 | + name: patches |
| 187 | + uid: "" |
| 188 | + blockOwnerDeletion: true |
| 189 | + controller: true |
0 commit comments