Merge pull request #3 from PDOK/PDOK-17806-mapserver-operator-blob-download-configmap

PDOK-17806 Setup blob download scripts in configmap

Author: damienmulder

api/v3/wms_types.go

@@ -31,6 +31,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"maps"
 	"slices"
+	"sort"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
@@ -154,21 +155,63 @@ func init() {
 	SchemeBuilder.Register(&WMS{}, &WMSList{})
 }
 
-func (wms *WMS) GetUniqueTiffBlobKeys() []string {
-	blobKeys := map[string]bool{}
+func (layer *Layer) getAllLayers() (layers []Layer) {
+	layers = append(layers, *layer)
+	if layer.Layers != nil {
+		for _, childLayer := range *layer.Layers {
+			layers = append(layers, childLayer.getAllLayers()...)
+		}
+	}
+	return
+}
+
+func (layer *Layer) hasData() bool {
+	switch {
+	case layer.Data == nil:
+		return false
+	case layer.Data.Gpkg != nil:
+		return true
+	case layer.Data.Postgis != nil:
+		return true
+	case layer.Data.TIF != nil:
+		return true
+	default:
+		return false
+	}
+}
 
-	if wms.Spec.Service.Layer.Data.TIF != nil && wms.Spec.Service.Layer.Data.TIF.BlobKey != "" {
-		blobKeys[wms.Spec.Service.Layer.Data.TIF.BlobKey] = true
+func (layer *Layer) hasTIFData() bool {
+	if !layer.hasData() {
+		return false
 	}
+	return layer.Data.TIF != nil && layer.Data.TIF.BlobKey != ""
+}
 
-	if wms.Spec.Service.Layer.Layers != nil && len(*wms.Spec.Service.Layer.Layers) > 0 {
-		for _, layer := range *wms.Spec.Service.Layer.Layers {
-			if layer.Data.TIF != nil && layer.Data.TIF.BlobKey != "" {
-				blobKeys[layer.Data.TIF.BlobKey] = true
+func (wms *WMS) GetAllLayersWithLegend() (layers []Layer) {
+	for _, layer := range wms.Spec.Service.Layer.getAllLayers() {
+		if !layer.hasData() || len(layer.Styles) == 0 {
+			continue
+		}
+		for _, style := range layer.Styles {
+			if style.Legend != nil && style.Legend.BlobKey != "" {
+				layers = append(layers, layer)
+				break
 			}
 		}
 	}
-	return slices.Collect(maps.Keys(blobKeys))
+	return
+}
+
+func (wms *WMS) GetUniqueTiffBlobKeys() []string {
+	blobKeys := map[string]bool{}
+	for _, layer := range wms.Spec.Service.Layer.getAllLayers() {
+		if layer.hasTIFData() {
+			blobKeys[layer.Data.TIF.BlobKey] = true
+		}
+	}
+	keys := slices.Collect(maps.Keys(blobKeys))
+	sort.Strings(keys) // This is only needed for the unit test
+	return keys
 }
 
 func (wms *WMS) GetAuthority() *Authority {

cmd/main.go

@@ -44,6 +44,10 @@ import (
 	// +kubebuilder:scaffold:imports
 )
 
+const (
+	defaultMultitoolImage = "docker.io/pdok/docker-multitool:0.9.1"
+)
+
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -67,6 +71,7 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var tlsOpts []func(*tls.Config)
+	var multitoolImage string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -84,6 +89,7 @@ func main() {
 	flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
+	flag.StringVar(&multitoolImage, "multitool-image", defaultMultitoolImage, "The image to use in the blob download init-container.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -213,8 +219,9 @@ func main() {
 		os.Exit(1)
 	}
 	if err = (&controller.WFSReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:         mgr.GetClient(),
+		Scheme:         mgr.GetScheme(),
+		MultitoolImage: multitoolImage,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "WFS")
 		os.Exit(1)

go.mod

@@ -7,11 +7,12 @@ godebug default=go1.23
 require (
 	github.com/onsi/ginkgo/v2 v2.21.0
 	github.com/onsi/gomega v1.35.1
-	github.com/pdok/smooth-operator v0.0.2
+	github.com/pdok/smooth-operator v0.0.4
 	k8s.io/api v0.32.0
 	k8s.io/apimachinery v0.32.0
 	k8s.io/client-go v0.32.0
 	sigs.k8s.io/controller-runtime v0.20.0
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -26,8 +27,9 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirectC
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
@@ -96,6 +98,7 @@ require (
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/kustomize/api v0.19.0 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.19.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -20,8 +20,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -30,6 +30,8 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/
 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -97,8 +99,8 @@ github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM
 github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
 github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
-github.com/pdok/smooth-operator v0.0.2 h1:RWaPpP4q6bFl3UG5aYkK/GIGeqGro7bQ/tPQuqOkync=
-github.com/pdok/smooth-operator v0.0.2/go.mod h1:Jz2AcpdfElRbjJsJNUuuk/o5XGZPZAC+820bVpRysbE=
+github.com/pdok/smooth-operator v0.0.4 h1:NVt7srlkSyiZooqn9R2pnMapY6jwZDVTj4QxUjy7UB0=
+github.com/pdok/smooth-operator v0.0.4/go.mod h1:oZWFuIKJGjN/C6ocgMNfMZ7SbLQi+N0qaWj7j95Wdec=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -128,8 +130,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -243,6 +245,10 @@ sigs.k8s.io/controller-runtime v0.20.0 h1:jjkMo29xEXH+02Md9qaVXfEIaMESSpy3TBWPrs
 sigs.k8s.io/controller-runtime v0.20.0/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
+sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ=
+sigs.k8s.io/kustomize/api v0.19.0/go.mod h1:/BbwnivGVcBh1r+8m3tH1VNxJmHSk1PzP5fkP6lbL1o=
+sigs.k8s.io/kustomize/kyaml v0.19.0 h1:RFge5qsO1uHhwJsu3ipV7RNolC7Uozc0jUBC/61XSlA=
+sigs.k8s.io/kustomize/kyaml v0.19.0/go.mod h1:FeKD5jEOH+FbZPpqUghBP8mrLjJ3+zD3/rf9NNu1cwY=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

internal/controller/blobdownload/blob_download.go

@@ -0,0 +1,143 @@
+package blobdownload
+
+import (
+	"fmt"
+	pdoknlv3 "github.com/pdok/mapserver-operator/api/v3"
+	"os"
+	"regexp"
+	"strings"
+)
+
+const (
+	scriptPath = "./gpkg_download.sh"
+	tifPath    = "/srv/data/tif"
+	imagesPath = "/srv/data/images"
+	fontsPath  = "/srv/data/config/fonts"
+	legendPath = "/var/www/legend"
+)
+
+func GetScript() (config string, err error) {
+	content, err := os.ReadFile(scriptPath)
+	if err != nil {
+		return "", err
+	}
+	return string(content), nil
+}
+
+func GetArgs[W pdoknlv3.WFS | pdoknlv3.WMS](webservice W) (args string, err error) {
+	var sb strings.Builder
+
+	switch any(webservice).(type) {
+	case pdoknlv3.WFS:
+		if WFS, ok := any(webservice).(pdoknlv3.WFS); ok {
+			createConfig(&sb)
+			downloadGeopackage(&sb, *WFS.Spec.Options.PrefetchData)
+			// In case of WFS no downloads are needed for TIFFs, styling assets and legends
+		}
+	case pdoknlv3.WMS:
+		if WMS, ok := any(webservice).(pdoknlv3.WMS); ok {
+			createConfig(&sb)
+			downloadGeopackage(&sb, *WMS.Spec.Options.PrefetchData)
+			if err = downloadTiffs(&sb, &WMS); err != nil {
+				return "", err
+			}
+			if err = downloadStylingAssets(&sb, &WMS); err != nil {
+				return "", err
+			}
+			if err = downloadLegends(&sb, &WMS); err != nil {
+				return "", err
+			}
+		}
+	default:
+		return "", fmt.Errorf("unexpected input, webservice should be of type WFS or WMS, webservice: %v", webservice)
+	}
+	return sb.String(), nil
+}
+
+func createConfig(sb *strings.Builder) {
+	writeLine(sb, "set -e;")
+	writeLine(sb, "mkdir -p /srv/data/config/;")
+	writeLine(sb, "rclone config create --non-interactive --obscure blobs azureblob endpoint $BLOBS_ENDPOINT account $BLOBS_ACCOUNT key $BLOBS_KEY use_emulator true;")
+}
+
+func downloadGeopackage(sb *strings.Builder, prefetchData bool) {
+	if prefetchData {
+		writeLine(sb, "bash /srv/scripts/gpkg_download.sh;")
+	}
+}
+
+func downloadTiffs(sb *strings.Builder, WMS *pdoknlv3.WMS) error {
+	if !*WMS.Spec.Options.PrefetchData {
+		return nil
+	}
+
+	for _, blobKey := range WMS.GetUniqueTiffBlobKeys() {
+		fileName, err := getFilenameFromBlobKey(blobKey)
+		if err != nil {
+			return err
+		}
+		writeLine(sb, "rclone copyto blobs:/%s  %s/%s || exit 1;", blobKey, tifPath, fileName)
+	}
+	return nil
+}
+
+func downloadStylingAssets(sb *strings.Builder, WMS *pdoknlv3.WMS) error {
+	for _, blobKey := range WMS.Spec.Service.StylingAssets.BlobKeys {
+		fileName, err := getFilenameFromBlobKey(blobKey)
+		if err != nil {
+			return err
+		}
+		path := imagesPath
+		isTTF, _ := regexp.MatchString(".*\\.(ttf)$", fileName)
+		if isTTF {
+			path = fontsPath
+		}
+		writeLine(sb, "rclone copyto blobs:/%s %s/%s || exit 1;", blobKey, path, fileName)
+		if isTTF {
+			fileRoot, err := getRootFromFilename(fileName)
+			if err != nil {
+				return err
+			}
+			writeLine(sb, "echo %s %s >> %s/fonts.list;", fileRoot, fileName, fontsPath)
+		}
+	}
+	writeLine(sb, "echo 'generated fonts.list:';")
+	writeLine(sb, "cat %v/fonts.list;", fontsPath)
+	return nil
+}
+
+func downloadLegends(sb *strings.Builder, WMS *pdoknlv3.WMS) error {
+	for _, layer := range WMS.GetAllLayersWithLegend() {
+		writeLine(sb, "mkdir -p %s/%s;", legendPath, layer.Name)
+		for _, style := range layer.Styles {
+			writeLine(sb, "rclone copyto blobs:/%s  %s/%s/%s.png || exit 1;", style.Legend.BlobKey, legendPath, layer.Name, style.Name)
+			fileName, err := getFilenameFromBlobKey(style.Legend.BlobKey)
+			if err != nil {
+				return err
+			}
+			writeLine(sb, "Copied legend %s to %s/%s/%s.png;", fileName, legendPath, layer.Name, style.Name)
+		}
+	}
+	writeLine(sb, "chown -R 999:999 %s", legendPath)
+	return nil
+}
+
+func getFilenameFromBlobKey(blobKey string) (string, error) {
+	index := strings.LastIndex(blobKey, "/")
+	if index == -1 {
+		return "", fmt.Errorf("could not determine filename from blobkey %s", blobKey)
+	}
+	return blobKey[index+1:], nil
+}
+
+func getRootFromFilename(fileName string) (string, error) {
+	index := strings.LastIndex(fileName, ".")
+	if index == -1 {
+		return "", fmt.Errorf("could not determine root from filename %s", fileName)
+	}
+	return fileName[:index], nil
+}
+
+func writeLine(sb *strings.Builder, format string, a ...any) {
+	sb.WriteString(fmt.Sprintf(format, a...) + "\n")
+}